Git fuzzing: uncomment the existing and add new targets. (#11486)

This pull request:
* Uncomments fuzzing targets that were commented out
* Adds the targets that were mentioned in the
[audit](https://ostif.org/wp-content/uploads/2023/01/X41-OSTIF-Gitlab-Git-Security-Audit-20230117-public.pdf)
done by X41
* Adds some new fuzzing targets
* Fixes false positives in the build with memory sanitizer because zlib
not being built with the sanitizer
* Builds corpus from smaller files
* Removes unnecessary memory allocations in the fuzzing harness
* Fixes compiler warnings (const unsigned char)

Author: JarLob

projects/git/Dockerfile

@@ -23,6 +23,7 @@ RUN apt-get update && \
         perl-modules libyaml-perl libz-dev python subversion tcl unzip \
         asciidoc docbook-xsl xmlto libssl-dev zip
 RUN git clone https://github.com/git/git git
+RUN git clone --depth 1 https://github.com/madler/zlib
 WORKDIR git
 RUN git checkout origin/next
 RUN git pull origin next

projects/git/Makefile.diff

@@ -1,32 +1,98 @@
 diff --git a/Makefile b/Makefile
-index 6bfb62cbe9..2230b40a9f 100644
+index 4e255c81f2..05905e3a2e 100644
 --- a/Makefile
 +++ b/Makefile
-@@ -756,6 +756,11 @@ FUZZ_OBJS += oss-fuzz/fuzz-commit-graph.o
+@@ -760,6 +760,19 @@ FUZZ_OBJS += oss-fuzz/fuzz-commit-graph.o
  FUZZ_OBJS += oss-fuzz/fuzz-date.o
  FUZZ_OBJS += oss-fuzz/fuzz-pack-headers.o
  FUZZ_OBJS += oss-fuzz/fuzz-pack-idx.o
 +FUZZ_OBJS += oss-fuzz/fuzz-command.o
 +FUZZ_OBJS += oss-fuzz/fuzz-cmd-status.o
 +FUZZ_OBJS += oss-fuzz/fuzz-cmd-version.o
 +FUZZ_OBJS += oss-fuzz/fuzz-cmd-diff.o
++FUZZ_OBJS += oss-fuzz/fuzz-credential-from-url-gently.o
++FUZZ_OBJS += oss-fuzz/fuzz-url-decode-mem.o
++FUZZ_OBJS += oss-fuzz/fuzz-url-end-with-slash.o
++FUZZ_OBJS += oss-fuzz/fuzz-parse-attr-line.o
++FUZZ_OBJS += oss-fuzz/fuzz-cmd-bundle-verify.o
++FUZZ_OBJS += oss-fuzz/fuzz-cmd-unpack-objects.o
++FUZZ_OBJS += oss-fuzz/fuzz-cmd-apply-check.o
++FUZZ_OBJS += oss-fuzz/fuzz-cmd-tag-create.o
 +
  .PHONY: fuzz-objs
  fuzz-objs: $(FUZZ_OBJS)
 
-@@ -1085,6 +1089,7 @@ LIB_OBJS += oid-array.o
+@@ -1087,6 +1100,7 @@ LIB_OBJS += oid-array.o
  LIB_OBJS += oidmap.o
  LIB_OBJS += oidset.o
  LIB_OBJS += oidtree.o
 +LIB_OBJS += oss-fuzz/fuzz-cmd-base.o
  LIB_OBJS += pack-bitmap-write.o
  LIB_OBJS += pack-bitmap.o
  LIB_OBJS += pack-check.o
+@@ -3862,10 +3876,10 @@ FUZZ_CXXFLAGS ?= $(ALL_CFLAGS)
+ 
+ .PHONY: fuzz-all
+ 
+-$(FUZZ_PROGRAMS): %: %.o oss-fuzz/dummy-cmd-main.o $(GITLIBS) GIT-LDFLAGS
+-	$(QUIET_LINK)$(CXX) $(FUZZ_CXXFLAGS) -o $@ $(ALL_LDFLAGS) \
++$(FUZZ_PROGRAMS): all
++	$(QUIET_LINK)$(CXX) $(FUZZ_CXXFLAGS) -o $@ $(ALL_LDFLAGS) $(BUILTIN_OBJS) \
+ 		-Wl,--allow-multiple-definition \
+-		$(filter %.o,$^) $(filter %.a,$^) $(LIBS) $(LIB_FUZZING_ENGINE)
++		$(filter %.o,$^) $(filter %.a,$^) git.o [email protected] $(LIBS) $(LIB_FUZZING_ENGINE)
+ 
+ fuzz-all: $(FUZZ_PROGRAMS)
+ 
+diff --git a/attr.c b/attr.c
+index 679e42258c..20e426726a 100644
+--- a/attr.c
++++ b/attr.c
+@@ -351,7 +351,7 @@ static const char *parse_attr(const char *src, int lineno, const char *cp,
+ 	return ep + strspn(ep, blank);
+ }
+ 
+-static struct match_attr *parse_attr_line(const char *line, const char *src,
++struct match_attr *parse_attr_line(const char *line, const char *src,
+ 					  int lineno, unsigned flags)
+ {
+ 	size_t namelen, num_attr, i;
+diff --git a/builtin/bundle.c b/builtin/bundle.c
+index 3ad11dc5d0..2443906572 100644
+--- a/builtin/bundle.c
++++ b/builtin/bundle.c
+@@ -123,7 +123,7 @@ static int open_bundle(const char *path, struct bundle_header *header,
+ 	return read_bundle_header(path, header);
+ }
+ 
+-static int cmd_bundle_verify(int argc, const char **argv, const char *prefix) {
++int cmd_bundle_verify(int argc, const char **argv, const char *prefix) {
+ 	struct bundle_header header = BUNDLE_HEADER_INIT;
+ 	int bundle_fd = -1;
+ 	int quiet = 0;
+diff --git a/builtin/tag.c b/builtin/tag.c
+index 37473ac21f..c719ea10e7 100644
+--- a/builtin/tag.c
++++ b/builtin/tag.c
+@@ -589,8 +589,11 @@ int cmd_tag(int argc, const char **argv, const char *prefix)
+ 	if (repo_get_oid(the_repository, object_ref, &object))
+ 		die(_("Failed to resolve '%s' as a valid ref."), object_ref);
+ 
+-	if (strbuf_check_tag_ref(&ref, tag))
+-		die(_("'%s' is not a valid tag name."), tag);
++	if (strbuf_check_tag_ref(&ref, tag)) {
++		//die(_("'%s' is not a valid tag name."), tag);
++		ret = 1;
++		goto cleanup;
++	}
+ 
+ 	if (read_ref(ref.buf, &prev))
+ 		oidclr(&prev);
 diff --git a/diff.c b/diff.c
-index 648f6717a5..24e42d0cd1 100644
+index ccfa1fca0d..97f895a122 100644
 --- a/diff.c
 +++ b/diff.c
-@@ -3557,7 +3557,7 @@ static void builtin_diff(const char *name_a,
+@@ -3654,7 +3654,7 @@ static void builtin_diff(const char *name_a,
  		}
  		if (fill_mmfile(o->repo, &mf1, one) < 0 ||
  		    fill_mmfile(o->repo, &mf2, two) < 0)
@@ -35,7 +101,25 @@ index 648f6717a5..24e42d0cd1 100644
  		/* Quite common confusing case */
  		if (mf1.size == mf2.size &&
  		    !memcmp(mf1.ptr, mf2.ptr, mf1.size)) {
-@@ -4009,7 +4009,7 @@ int diff_populate_filespec(struct repository *r,
+@@ -3837,7 +3837,7 @@ static void builtin_diffstat(const char *name_a, const char *name_b,
+ 
+ 		if (fill_mmfile(o->repo, &mf1, one) < 0 ||
+ 		    fill_mmfile(o->repo, &mf2, two) < 0)
+-			die("unable to read files to diff");
++			return;//die("unable to read files to diff");
+ 
+ 		memset(&xpp, 0, sizeof(xpp));
+ 		memset(&xecfg, 0, sizeof(xecfg));
+@@ -3900,7 +3900,7 @@ static void builtin_checkdiff(const char *name_a, const char *name_b,
+ 
+ 	if (fill_mmfile(o->repo, &mf1, one) < 0 ||
+ 	    fill_mmfile(o->repo, &mf2, two) < 0)
+-		die("unable to read files to diff");
++		return;//die("unable to read files to diff");
+ 
+ 	/*
+ 	 * All the other codepaths check both sides, but not checking
+@@ -4106,7 +4106,7 @@ int diff_populate_filespec(struct repository *r,
  		conv_flags = CONV_EOL_RNDTRP_WARN;
 
  	if (!DIFF_FILE_VALID(s))
@@ -44,7 +128,7 @@ index 648f6717a5..24e42d0cd1 100644
  	if (S_ISDIR(s->mode))
  		return -1;
 
-@@ -4113,7 +4113,7 @@ int diff_populate_filespec(struct repository *r,
+@@ -4210,7 +4210,7 @@ int diff_populate_filespec(struct repository *r,
  		}
  		if (oid_object_info_extended(r, &s->oid, &info,
  					     OBJECT_INFO_LOOKUP_REPLACE))
@@ -53,7 +137,7 @@ index 648f6717a5..24e42d0cd1 100644
 
  object_read:
  		if (size_only || check_binary) {
-@@ -4128,7 +4128,7 @@ int diff_populate_filespec(struct repository *r,
+@@ -4225,7 +4225,7 @@ int diff_populate_filespec(struct repository *r,
  			info.contentp = &s->data;
  			if (oid_object_info_extended(r, &s->oid, &info,
  						     OBJECT_INFO_LOOKUP_REPLACE))
@@ -62,7 +146,7 @@ index 648f6717a5..24e42d0cd1 100644
  		}
  		s->should_free = 1;
  	}
-@@ -7051,7 +7051,7 @@ size_t fill_textconv(struct repository *r,
+@@ -7226,7 +7226,7 @@ size_t fill_textconv(struct repository *r,
  			return 0;
  		}
  		if (diff_populate_filespec(r, df, NULL))
@@ -71,7 +155,7 @@ index 648f6717a5..24e42d0cd1 100644
  		*outbuf = df->data;
  		return df->size;
  	}
-@@ -7069,7 +7069,7 @@ size_t fill_textconv(struct repository *r,
+@@ -7244,7 +7244,7 @@ size_t fill_textconv(struct repository *r,
 
  	*outbuf = run_textconv(r, driver->textconv, df, &size);
  	if (!*outbuf)
@@ -81,10 +165,10 @@ index 648f6717a5..24e42d0cd1 100644
  	if (driver->textconv_cache && df->oid_valid) {
  		/* ignore errors, as we might be in a readonly repository */
 diff --git a/environment.c b/environment.c
-index 18d042b467..7e0ba9dc47 100644
+index 90632a39bc..29e7c0479f 100644
 --- a/environment.c
 +++ b/environment.c
-@@ -265,7 +265,7 @@ void set_git_work_tree(const char *new_work_tree)
+@@ -271,7 +271,7 @@ void set_git_work_tree(const char *new_work_tree)
  		strbuf_release(&realpath);
  		return;
  	}

projects/git/build.sh

@@ -15,6 +15,14 @@
 #
 ################################################################################
 
+# build zlib
+pushd "$SRC/zlib"
+./configure --static --prefix="$WORK"
+make -j$(nproc) CFLAGS="$CFLAGS -fPIC"
+make install
+popd
+export ZLIB_PATH=$WORK
+
 # Enable a timeout for lockfiles rather than exit immediately. This is to
 # overcome in case multiple processes try to lock a file around the same
 # time. 
@@ -25,34 +33,67 @@ make -j$(nproc) CC=$CC CXX=$CXX CFLAGS="$CFLAGS" \
   FUZZ_CXXFLAGS="$CXXFLAGS -Wl,--allow-multiple-definition" \
   LIB_FUZZING_ENGINE="common-main.o $LIB_FUZZING_ENGINE" fuzz-all
 
-FUZZERS="fuzz-pack-headers fuzz-pack-idx fuzz-commit-graph"
-#FUZZERS="$FUZZERS fuzz-cmd-status fuzz-cmd-diff fuzz-cmd-version"
-#FUZZERS="$FUZZERS fuzz-command"
+FUZZERS=""
+# FUZZERS="$FUZZERS fuzz-cmd-apply-check"
+FUZZERS="$FUZZERS fuzz-cmd-bundle-verify"
 FUZZERS="$FUZZERS fuzz-cmd-diff"
+# FUZZERS="$FUZZERS fuzz-cmd-status"
+# FUZZERS="$FUZZERS fuzz-cmd-tag-create"
+# FUZZERS="$FUZZERS fuzz-cmd-unpack-objects"
+# FUZZERS="$FUZZERS fuzz-cmd-version"
+# FUZZERS="$FUZZERS fuzz-command"
+FUZZERS="$FUZZERS fuzz-commit-graph"
+FUZZERS="$FUZZERS fuzz-credential-from-url-gently"
+FUZZERS="$FUZZERS fuzz-date"
+FUZZERS="$FUZZERS fuzz-pack-headers"
+FUZZERS="$FUZZERS fuzz-pack-idx"
+FUZZERS="$FUZZERS fuzz-parse-attr-line"
+FUZZERS="$FUZZERS fuzz-url-decode-mem"
+FUZZERS="$FUZZERS fuzz-url-end-with-slash"
+
 # copy fuzzers
 for fuzzer in $FUZZERS ; do
   cp oss-fuzz/$fuzzer $OUT
 done
 
-# build corpora from Git's own packfiles
+# build commit-graph corpus
+ASAN_OPTIONS=detect_leaks=0 ./git commit-graph write
+zip -j $OUT/fuzz-commit-graph_seed_corpus .git/objects/info/commit-graph
+
+# Git's own packfiles are too big for effective fuzzing
+# build corpora from a new repository
+mkdir mock-repo
+pushd mock-repo
+../git init
+echo "abc" > TEMP_1
+../git add .
+../git config user.email "[email protected]"
+../git config user.name "Your Name"
+../git commit -m "initial commit"
+../git repack
 zip -j $OUT/fuzz-pack-idx_seed_corpus.zip .git/objects/pack/*.idx
+zip -j $OUT/fuzz-cmd-unpack-objects_seed_corpus .git/objects/pack/*.pack
 for packfile in .git/objects/pack/*.pack ; do
   dd ibs=1 skip=12 if=$packfile of=$packfile.trimmed
 done
 zip -j $OUT/fuzz-pack-headers_seed_corpus.zip .git/objects/pack/*.pack.trimmed
+ASAN_OPTIONS=detect_leaks=0 ../git bundle create test.bundle master
+zip -j $OUT/fuzz-cmd-bundle-verify_seed_corpus test.bundle
+echo "adc\nrb\n" > TEMP_1
+../git diff > test.patch
+zip -j $OUT/fuzz-cmd-apply-check_seed_corpus test.patch
+popd
+rm -rf mock-repo
 
-# build commit-graph corpus
-ASAN_OPTIONS=detect_leaks=0 ./git commit-graph write
-zip -j $OUT/fuzz-commit-graph_seed_corpus .git/objects/info/commit-graph
-
-# Mute stderr
 for fuzzer in $FUZZERS ; do
   cat >$OUT/$fuzzer.options << EOF
 [libfuzzer]
 detect_leaks = 0
 EOF
 done
 
+echo -e "max_len = 250\n" >> $OUT/fuzz-cmd-tag-create.options
+
 # Generate existing file for temp git repository
 echo "TEMP1TEMP1TEMP1TEMP1" > $OUT/TEMP_1
 echo "TEMP2TEMP2TEMP2TEMP2" > $OUT/TEMP_2

projects/git/fuzz-cmd-apply-check.c

@@ -0,0 +1,61 @@
+/* Copyright 2024 Google LLC
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+	  http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+#include <stddef.h>
+#include <stdint.h>
+#include <unistd.h>
+#include "git-compat-util.h"
+#include "repository.h"
+#include "fuzz-cmd-base.h"
+
+int cmd_init_db(int argc, const char **argv, const char *prefix);
+int cmd_apply(int argc, const char **argv, const char *prefix);
+
+static int initialized = 0;
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
+{
+	char *argv[3];
+
+	if (size < 1)
+		return 0;
+
+	if (!initialized)
+	{
+		put_envs();
+		create_templ_dir();
+	}
+
+	initialize_the_repository();
+
+	if (!initialized)
+	{
+		argv[0] = "init";
+		argv[1] = "--quiet";
+		argv[2] = NULL;
+		if (cmd_init_db(2, (const char **)argv, (const char *)""))
+			exit(1);
+
+		initialized = 1;
+	}
+
+	if (-1 == write(STDIN_FILENO, data, size))
+		goto cleanup;
+
+	argv[0] = "apply";
+	argv[1] = "-check";
+	argv[2] = NULL;
+	cmd_apply(2, (const char **)argv, (const char *)"");
+
+cleanup:
+	repo_clear(the_repository);
+	return 0;
+}