Basic manual sha calculation works

Author: cirosantilli

generate-commits-fast.md

@@ -9,5 +9,6 @@ It is sometimes interesting to generate a ton of commits to test some edge case,
 - `openssl dgst -sha1`: 22s
 - `git hash-object --stdin -w`: 21s
 - `git hash-object --stdin`: 20s
-- `sha1sum` Coreutils: 1.4s. TODO: why so much faster than `hash-object`? This is minimum bottleneck per CPU. We can reuse identical blobs. 
-- touch: 0.9s (same on ramfs). This is the minimum IO bottleneck. Since the CPU bottleneck is not much above, parallelization is not the trouble.
+- `sha1sum` Coreutils: 1.4s.
+- touch: 0.9s (same on ramfs).
+- `time python3 <(printf 'import hashlib; import sys;\nfor i in range(1000): print(hashlib.sha1(str(i).encode("ascii")).hexdigest())')`: 0.14s TODO: why so much faster than `hash-object`? This is minimum bottleneck per CPU.

other-test-repos/.gitignore

@@ -1,2 +1,3 @@
-/clone/
-/repo/
+*.pyc
+/*.tmp/
+__pycache__/

other-test-repos/commit-meta.bashrc

@@ -2,9 +2,9 @@
 date='2000-01-01T00:00:00+0000'
 email='[email protected]'
 name='a'
-export GIT_COMMITTER_EMAIL="$email"
-export GIT_COMMITTER_NAME="$name"
-export GIT_COMMITTER_DATE="$date"
+export GIT_AUTHOR_DATE="$date"
 export GIT_AUTHOR_EMAIL="$email"
 export GIT_AUTHOR_NAME="$name"
-export GIT_AUTHOR_DATE="$date"
+export GIT_COMMITTER_DATE="$date"
+export GIT_COMMITTER_EMAIL="$email"
+export GIT_COMMITTER_NAME="$name"

other-test-repos/finish-bare.bashrc

@@ -1 +1 @@
-git clone -q repo clone
+git clone -q repo.tmp clone.tmp

other-test-repos/init.bashrc

@@ -1,4 +1,4 @@
-rm -rf repo clone
-mkdir -p repo
-cd repo
+rm -rf repo.tmp clone.tmp
+mkdir -p repo.tmp
+cd repo.tmp
 git init -q

other-test-repos/manual-sha-calculation.py

@@ -1,14 +1,86 @@
 #!/usr/bin/env python3
 
+"""
+Create objects very manually. Goals:
+
+- learn the object file
+- be faster than Git to generate large repos
+"""
+
 import datetime
+import subprocess
+import hashlib
+import zlib
 import os
 
-date0 = datetime.date(2000, 1, 1)
-datef = datetime.date(2100, 1, 1)
-date = date0
-while date < datef:
-    s = date.strftime("%Y-%m-%dT01:00:00")
-    cmd = 'GIT_COMMITTER_DATE="{0}" git commit --allow-empty --allow-empty-message -m "" --date="{0}"'.format(s)
-    print cmd
-    os.system(cmd)
-    date += datetime.timedelta(days=1)
+import util
+
+util.init()
+git_dir = b'.git'
+objects_dir = os.path.join(git_dir, b'objects')
+
+# Directory parameters.
+blob_content = b'a'
+blob_basename = b'a'
+blob_mode = b'100644'
+
+name = b'a'
+email = b'[email protected]'
+# 2000-01-01T00:00:00+0000
+date = b'946684800 +0000'
+
+author_date = date
+author_email = email
+author_name = name
+committer_date = date
+committer_email = email
+committer_name = name
+message = b'a'
+
+def get_object_and_sha(obj_type, content):
+    obj = b'%s %s\0%s' % (obj_type, str(len(content)).encode('ascii'), content)
+    hash = hashlib.sha1(obj)
+    return (obj, hash.hexdigest().encode('ascii'), hash.digest())
+
+def save_object(obj_type, content):
+    obj, sha_ascii, sha = get_object_and_sha(obj_type, content)
+    obj_dir = os.path.join(objects_dir, sha_ascii[:2])
+    obj_path = os.path.join(obj_dir, sha_ascii[2:])
+    os.makedirs(obj_dir, exist_ok=True)
+    with open(obj_path, 'wb') as f:
+        f.write(zlib.compress(obj))
+
+def get_git_hash_object(obj_type, input):
+    cmd = [b'git', b'hash-object', b'--stdin', b'-t', obj_type]
+    return subprocess.check_output(cmd, input=input).rstrip()
+
+# Blob.
+save_object(b'blob', blob_content)
+obj, blob_sha_ascii, blob_sha = get_object_and_sha(b'blob', blob_content)
+# Check sha matches Git.
+blob_sha_git = get_git_hash_object(b'blob', blob_content)
+assert blob_sha_ascii == blob_sha_git
+
+# Tree.
+tree_content = b'%s %s\0%s' % (blob_mode, blob_basename, blob_sha)
+save_object(b'tree', tree_content)
+# Check sha matches Git.
+obj, tree_sha_ascii, tree_sha = get_object_and_sha(b'tree', tree_content)
+tree_sha_git = get_git_hash_object(b'tree', tree_content)
+assert tree_sha_ascii == tree_sha_git
+
+# Commit.
+commit_content = b'tree %s\nauthor %s <%s> %s\ncommitter %s <%s> %s\n\n%s\n' % (
+        tree_sha_ascii,
+        author_name, author_email, author_date,
+        committer_name, committer_email, committer_date,
+        message)
+save_object(b'commit', commit_content)
+# Check sha matches Git.
+obj, commit_sha_ascii, commit_sha = get_object_and_sha(b'commit', commit_content)
+commit_sha_git = get_git_hash_object(b'commit', commit_content)
+assert commit_sha_ascii == commit_sha_git
+
+# Create master branch.
+subprocess.check_output(['git', 'branch', 'master', commit_sha_ascii])
+subprocess.check_output(['git', 'clone', '.', '../clone.tmp'])

other-test-repos/streak.py

@@ -2,17 +2,28 @@
 
 import datetime
 import os
+import shutil
+import subprocess
 
-repo = 'repo'
-os.mkdir(repo)
-os.cwd(repo)
+import util
+
+util.init()
+
+name = 'a'
+email = '[email protected]'
+os.environ['GIT_AUTHOR_EMAIL'] = email
+os.environ['GIT_AUTHOR_NAME'] = name
+os.environ['GIT_COMMITTER_EMAIL'] = email
+os.environ['GIT_COMMITTER_NAME'] = name
 
 date0 = datetime.date(2000, 1, 1)
 datef = datetime.date(2100, 1, 1)
 date = date0
 while date < datef:
     s = date.strftime('%Y-%m-%dT01:00:00')
-    cmd = 'GIT_COMMITTER_DATE="{0}" git commit --allow-empty --allow-empty-message -m "" --date="{0}"'.format(s)
-    print(cmd)
-    os.system(cmd)
+    print(s)
+    os.environ['GIT_AUTHOR_DATE'] = s
+    os.environ['GIT_COMMITTER_DATE'] = s
+    cmd = ['git', 'commit', '-q', '--allow-empty', '--allow-empty-message', '-m', '']
+    subprocess.check_output(cmd)
     date += datetime.timedelta(days=1)

other-test-repos/util.py

@@ -0,0 +1,11 @@
+import os
+import shutil
+import subprocess
+
+def init():
+    repo = 'repo.tmp'
+    for d in (repo, 'clone.tmp'):
+        shutil.rmtree(d, ignore_errors=True)
+    os.mkdir(repo)
+    os.chdir(repo)
+    subprocess.check_output(['git', 'init', '-q'])