Merge pull request #7 from HalfVoxel/master

S3 Improvements, fixes for large numbers of tracked files and support for --short status output

Author: alebedev

README.md

@@ -36,14 +36,15 @@ or in `clone/.git/config` (for per-repo settings).
 ```ini
 [git-media]
 transport = <scp|local|s3|atmos|webdav>
+autodownload = <true|false>
 
 # settings for scp transport
 scpuser = <user>
 scphost = <host>
 scppath = <path_on_remote_server>
 
 # settings for local transport
-path = <local_filesystem_path>
+localpath = <local_filesystem_path>
 
 # settings for s3 transport
 s3bucket = <name_of_bucket>
@@ -63,7 +64,6 @@ or in `clone/.git/config` (for per-repo settings).
 webdavverifyserver = <Net::Dav.verify_server setting, true by default>
 webdavbinarytransfer = <Net::Dav.new :curl option value, false by default>
 
-
 ```
 
 
@@ -89,8 +89,11 @@ you need to explicitly tell git that some media files has changed:
 
 
 ## Config Settings
+
+If autodownload is set to true, required files will automatically be
+downloaded when checking out or pulling. Default is false
 
-$ git config --global media.auto-download false
+$ git config --global media.autodownload true
 
 
 ## Installing

lib/git-media.rb

@@ -123,6 +123,10 @@ def self.get_pull_transport
  module Application
  def self.run!
 
+ if !system('git rev-parse')
+ return
+ end
+ 
  cmd = ARGV.shift # get the subcommand
  cmd_opts = case cmd
  when "filter-clean" # parse delete options
@@ -139,16 +143,20 @@ def self.run!
  GitMedia::Sync.run!
  when 'status'
  require 'git-media/status'
- Trollop::options do
+ opts = Trollop::options do
  opt :force, "Force status"
+ opt :short, "Short status"
  end
- GitMedia::Status.run!
+ GitMedia::Status.run!(opts)
  else
  print <<EOF
 usage: git media sync|status|clear
 
  sync	Sync files with remote server
+
  status	Show files that are waiting to be uploaded and file size
+ --short: Displays a shorter status message
+
  clear	Upload and delete the local cache of media files
 
 EOF

lib/git-media/clear.rb

@@ -9,12 +9,12 @@ def self.run!
  end
 
  def self.clear_local_cache
- # find files in media buffer and upload them
+ # find files in media buffer and delete all pushed files
  all_cache = Dir.chdir(GitMedia.get_media_buffer) { Dir.glob('*') }
  unpushed_files = @push.get_unpushed(all_cache)
  pushed_files = all_cache - unpushed_files
  pushed_files.each do |sha|
- puts "removing " + sha[0, 8]
+ puts "Removing " + sha[0, 8]
  File.unlink(File.join(GitMedia.get_media_buffer, sha))
  end
  end

lib/git-media/filter-clean.rb

@@ -6,34 +6,60 @@ module GitMedia
  module FilterClean
 
  def self.run!
- # determine and initialize our media buffer directory
- media_buffer = GitMedia.get_media_buffer
-
- hashfunc = Digest::SHA1.new
- start = Time.now
-
- # TODO: read first 41 bytes and see if this is a stub
 
- # read in buffered chunks of the data
- # calculating the SHA and copying to a tempfile
- tempfile = Tempfile.new('media')
- while data = STDIN.read(4096)
- hashfunc.update(data)
- tempfile.write(data)
- end
- tempfile.close
+ # Read first 42 bytes
+ # If the file is only 41 bytes long (as in the case of a stub)
+ # it will only return a string with a length of 41
+ data = STDIN.read(42)
 
- # calculate and print the SHA of the data
- STDOUT.print hx = hashfunc.hexdigest 
  STDOUT.binmode
- STDOUT.write("\n")
 
- # move the tempfile to our media buffer area
- media_file = File.join(media_buffer, hx)
- FileUtils.mv(tempfile.path, media_file)
+ if data != nil && data.length == 41 && data.match(/^[0-9a-fA-F]+\n$/)
+ 
+ # Exactly 41 bytes long and matches the hex string regex
+ # This is most likely a stub
+ # TODO: Maybe add some additional marker in the files like
+ # "[hex string]:git-media"
+ # to really be able to say that a file is a stub
+
+ STDOUT.write (data)
+ STDERR.puts("Skipping unexpanded stub : " + data[0, 8])
+
+ else
+
+ # determine and initialize our media buffer directory
+ media_buffer = GitMedia.get_media_buffer
+
+ hashfunc = Digest::SHA1.new
+ start = Time.now
 
- elapsed = Time.now - start
- STDERR.puts('Saving media : ' + hx + ' : ' + elapsed.to_s)
+ # read in buffered chunks of the data
+ # calculating the SHA and copying to a tempfile
+ tempfile = Tempfile.new('media', :binmode => true)
+
+ # Write the first 42 bytes
+ if data != nil
+ hashfunc.update(data)
+ tempfile.write(data)
+ end
+
+ while data = STDIN.read(4096)
+ hashfunc.update(data)
+ tempfile.write(data)
+ end
+ tempfile.close
+
+ # calculate and print the SHA of the data
+ STDOUT.print hx = hashfunc.hexdigest 
+ STDOUT.write("\n")
+
+ # move the tempfile to our media buffer area
+ media_file = File.join(media_buffer, hx)
+ FileUtils.mv(tempfile.path, media_file)
+
+ elapsed = Time.now - start
+ STDERR.puts('Saving media : ' + hx + ' : ' + elapsed.to_s)
+ end
  end
 
  end

lib/git-media/filter-smudge.rb

@@ -1,36 +1,65 @@
 module GitMedia
  module FilterSmudge
 
+ def self.print_stream(stream)
+ while data = stream.read(4096) do
+ print data
+ end
+ end
+
  def self.run!
  media_buffer = GitMedia.get_media_buffer
- can_download = false # TODO: read this from config and implement
 
  # read checksum size
- sha = STDIN.readline(64).strip # read no more than 64 bytes
+ orig = STDIN.readline(64)
+ sha = orig.strip # read no more than 64 bytes
  if STDIN.eof? && sha.length == 40 && sha.match(/^[0-9a-fA-F]+$/) != nil
  # this is a media file
  media_file = File.join(media_buffer, sha.chomp)
  if File.exists?(media_file)
- STDERR.puts('recovering media : ' + sha)
- File.open(media_file, 'r') do |f|
- while data = f.read(4096) do
- print data
- end
+ STDERR.puts('Recovering media : ' + sha)
+ File.open(media_file, 'rb') do |f|
+ print_stream(f)
  end
  else
- # TODO: download file if not in the media buffer area
- if !can_download
- STDERR.puts('media missing, saving placeholder : ' + sha)
- puts sha
+ # Read key from config
+ auto_download = `git config git-media.autodownload`.chomp.downcase == "true"
+
+ if auto_download
+
+ pull = GitMedia.get_pull_transport
+
+ cache_file = GitMedia.media_path(sha)
+ if !File.exist?(cache_file)
+ STDERR.puts ("Downloading : " + sha[0,8])
+ # Download the file from backend storage
+ # We have no idea what the final file will be (therefore nil)
+ pull.pull(nil, sha)
+ end
+
+ STDERR.puts ("Expanding : " + sha[0,8])
+
+ if File.exist?(cache_file)
+ File.open(media_file, 'rb') do |f|
+ print_stream(f)
+ end
+ else
+ STDERR.puts ("Could not get media, saving placeholder : " + sha)
+ puts orig
+ end
+
+ else
+ STDERR.puts('Media missing, saving placeholder : ' + sha)
+ # Print orig and not sha to preserve eventual newlines at end of file
+ # To avoid git thinking the file has changed
+ puts orig
  end
  end
  else
  # if it is not a 40 character long hash, just output
  STDERR.puts('Unknown git-media file format')
- print sha
- while data = STDIN.read(4096)
- print data
- end
+ print orig
+ print_stream(STDIN)
  end
  end
 

lib/git-media/status.rb

@@ -4,12 +4,12 @@
 module GitMedia
  module Status
 
- def self.run!
+ def self.run!(opts)
  @push = GitMedia.get_push_transport
  r = self.find_references
- self.print_references(r)
+ self.print_references(r, opts[:short])
  r = self.local_cache_status
- self.print_cache_status(r)
+ self.print_cache_status(r, opts[:short])
  end
 
  # find tree entries that are likely media references
@@ -19,7 +19,9 @@ def self.find_references
  files = files.map { |f| s = f.split("\t"); [s[0].split(' ').last, s[1]] }
  files = files.select { |f| f[0] == '41' } # it's the right size
  files.each do |tree_size, fname|
- if size = File.size?(fname)
+ if File.exists?(fname)
+ size = File.size(fname)
+
  # Windows newlines can offset file size by 1
  if size == tree_size.to_i or size == tree_size.to_i + 1
  # TODO: read in the data and verify that it's a sha + newline
@@ -39,49 +41,70 @@ def self.find_references
  references
  end
 
- def self.print_references(refs)
+ def self.print_references(refs, short=false)
+
  if refs[:to_expand].size > 0
  puts "== Unexpanded Media =="
- refs[:to_expand].each do |file, sha|
- puts " " + sha[0, 8] + " " + file
+ if short
+ puts "Count: " + refs[:to_expand].size.to_s
+ else
+ refs[:to_expand].each do |file, sha|
+ puts " " + sha[0, 8] + " " + file
+ end
+ puts
  end
- puts
  end
  if refs[:expanded].size > 0
  puts "== Expanded Media =="
- refs[:expanded].each do |file|
- size = File.size(file)
- puts " " + "(#{self.to_human(size)})".ljust(8) + " #{file}"
+ if short
+ puts "Count: " + refs[:expanded].size.to_s
+ else
+ refs[:expanded].each do |file|
+ size = File.size(file)
+ puts " " + "(#{self.to_human(size)})".ljust(8) + " #{file}"
+ end
+ puts
  end
- puts
  end
  if refs[:deleted].size > 0
  puts "== Deleted Media =="
- refs[:deleted].each do |file|
- puts " " + " #{file}"
+ if short
+ puts "Count: " + refs[:deleted].size.to_s
+ else
+ refs[:deleted].each do |file|
+ puts " " + " #{file}"
+ end
+ puts
  end
- puts
  end
  end
 
- def self.print_cache_status(refs)
+ def self.print_cache_status(refs, short)
  if refs[:unpushed].size > 0
  puts "== Unpushed Media =="
- refs[:unpushed].each do |sha|
- cache_file = GitMedia.media_path(sha)
- size = File.size(cache_file)
- puts " " + "(#{self.to_human(size)})".ljust(8) + ' ' + sha[0, 8]
+ if short
+ puts "Count: " + refs[:unpushed].size.to_s
+ else
+ refs[:unpushed].each do |sha|
+ cache_file = GitMedia.media_path(sha)
+ size = File.size(cache_file)
+ puts " " + "(#{self.to_human(size)})".ljust(8) + ' ' + sha[0, 8]
+ end
+ puts
  end
- puts
  end
  if refs[:pushed].size > 0
  puts "== Already Pushed Media =="
- refs[:pushed].each do |sha|
- cache_file = GitMedia.media_path(sha)
- size = File.size(cache_file)
- puts " " + "(#{self.to_human(size)})".ljust(8) + ' ' + sha[0, 8]
+ if short
+ puts "Count: " + refs[:pushed].size.to_s
+ else
+ refs[:pushed].each do |sha|
+ cache_file = GitMedia.media_path(sha)
+ size = File.size(cache_file)
+ puts " " + "(#{self.to_human(size)})".ljust(8) + ' ' + sha[0, 8]
+ end
+ puts
  end
- puts
  end
  end