diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 907698d..e580849 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -5,6 +5,9 @@ name: Python package on: [push, pull_request, workflow_dispatch] +permissions: + contents: read + jobs: build: @@ -12,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t"] os: [ubuntu-latest] experimental: [false] include: @@ -22,11 +25,11 @@ jobs: continue-on-error: ${{ matrix.experimental }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} allow-prereleases: ${{ matrix.experimental }} diff --git a/Makefile b/Makefile index a0a2d0e..20436bb 100644 --- a/Makefile +++ b/Makefile @@ -1,40 +1,12 @@ -PYTHON = python3 -SETUP = $(PYTHON) setup.py -TESTFLAGS = +.PHONY: all clean release force_release -all:: +all: @grep -Ee '^[a-z].*:' Makefile | cut -d: -f1 | grep -vF all -release:: clean - # Check if latest tag is the current head we're releasing - echo "Latest tag = $$(git tag | sort -nr | head -n1)" - echo "HEAD SHA = $$(git rev-parse head)" - echo "Latest tag SHA = $$(git tag | sort -nr | head -n1 | xargs git rev-parse)" - @test "$$(git rev-parse head)" = "$$(git tag | sort -nr | head -n1 | xargs git rev-parse)" - make force_release +clean: + rm -rf build/ dist/ .eggs/ .tox/ -force_release:: clean - git push --tags - python3 -m build --sdist --wheel +force_release: clean + ./build-release.sh twine upload dist/* - -doc:: - make -C doc/ html - -build:: - $(SETUP) build - $(SETUP) build_ext -i - -build_ext:: - $(SETUP) build_ext -i - -install:: - $(SETUP) install - -clean:: - $(SETUP) clean --all - rm -f *.so - -coverage:: build - PYTHONPATH=. $(PYTHON) -m pytest --cov=gitdb gitdb - + git push --tags origin master diff --git a/README.md b/README.md new file mode 100644 index 0000000..37a6e4e --- /dev/null +++ b/README.md @@ -0,0 +1,71 @@ +# Archive Notice + +The entire history of this repository is now contained in https://github.com/gitpython-developers/GitPython, where it is now maintained. +For issues, please use its issue tracker as well. + +# GitDB + +GitDB allows you to access bare git repositories for reading and writing. It aims at allowing full access to loose objects as well as packs with performance and scalability in mind. It operates exclusively on streams, allowing to handle large objects with a small memory footprint. + +## Installation + +[![Latest Version](https://img.shields.io/pypi/v/gitdb.svg)](https://pypi.python.org/pypi/gitdb/) +[![Supported Python versions](https://img.shields.io/pypi/pyversions/gitdb.svg)](https://pypi.python.org/pypi/gitdb/) +[![Documentation Status](https://readthedocs.org/projects/gitdb/badge/?version=latest)](https://readthedocs.org/projects/gitdb/?badge=latest) + +From [PyPI](https://pypi.python.org/pypi/gitdb): + +```shell +pip install gitdb +``` + +## Speedups + +If you want to go up to 20% faster, you can install gitdb-speedups with: + +```shell +pip install gitdb-speedups +``` + +However, please note that gitdb-speedups is not currently maintained. + +## Requirements + +* smmap — declared as a dependency, automatically installed +* pytest — for running the tests + +## Source + +The source is available in a git repository on GitHub: + +https://github.com/gitpython-developers/gitdb + +Once the clone is complete, please be sure to initialize the submodule using: + +```shell +cd gitdb +git submodule update --init +``` + +Run the tests with: + +```shell +pytest +``` + +## Development + +[![Python package](https://github.com/gitpython-developers/gitdb/workflows/Python%20package/badge.svg)](https://github.com/gitpython-developers/gitdb/actions) + +The library is considered mature, and not under active development. Its primary (known) use is in GitPython. + +## Infrastructure + +* Discussions + * https://github.com/gitpython-developers/GitPython/discussions +* Issue Tracker + * https://github.com/gitpython-developers/gitdb/issues + +## License + +New BSD License diff --git a/README.rst b/README.rst deleted file mode 100644 index 61ce28b..0000000 --- a/README.rst +++ /dev/null @@ -1,74 +0,0 @@ -GitDB -===== - -GitDB allows you to access bare git repositories for reading and writing. It aims at allowing full access to loose objects as well as packs with performance and scalability in mind. It operates exclusively on streams, allowing to handle large objects with a small memory footprint. - -Installation -============ - -.. image:: https://img.shields.io/pypi/v/gitdb.svg - :target: https://pypi.python.org/pypi/gitdb/ - :alt: Latest Version -.. image:: https://img.shields.io/pypi/pyversions/gitdb.svg - :target: https://pypi.python.org/pypi/gitdb/ - :alt: Supported Python versions -.. image:: https://readthedocs.org/projects/gitdb/badge/?version=latest - :target: https://readthedocs.org/projects/gitdb/?badge=latest - :alt: Documentation Status - -From `PyPI `_:: - - pip install gitdb - -SPEEDUPS -======== - -If you want to go up to 20% faster, you can install gitdb-speedups with:: - - pip install gitdb-speedups - -However, please note that gitdb-speedups is not currently maintained. - -REQUIREMENTS -============ - -* smmap - declared as a dependency, automatically installed -* pytest - for running the tests - -SOURCE -====== - -The source is available in a git repository on GitHub: - -https://github.com/gitpython-developers/gitdb - -Once the clone is complete, please be sure to initialize the submodule using:: - - cd gitdb - git submodule update --init - -Run the tests with:: - - pytest - -DEVELOPMENT -=========== - -.. image:: https://github.com/gitpython-developers/gitdb/workflows/Python%20package/badge.svg - :target: https://github.com/gitpython-developers/gitdb/actions - -The library is considered mature, and not under active development. Its primary (known) use is in GitPython. - -INFRASTRUCTURE -============== - -* Discussions - * https://github.com/gitpython-developers/GitPython/discussions - -* Issue Tracker - * https://github.com/gitpython-developers/gitdb/issues - -LICENSE -======= - -New BSD License diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..95389ff --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,3 @@ +# Security Policy + +See [GitPython](https://github.com/gitpython-developers/GitPython/blob/main/SECURITY.md). Vulnerabilities found in `gitdb` can be reported there. diff --git a/build-release.sh b/build-release.sh new file mode 100755 index 0000000..5840e44 --- /dev/null +++ b/build-release.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# +# This script builds a release. If run in a venv, it auto-installs its tools. +# You may want to run "make release" instead of running this script directly. + +set -eEu + +function release_with() { + $1 -m build --sdist --wheel +} + +if test -n "${VIRTUAL_ENV:-}"; then + deps=(build twine) # Install twine along with build, as we need it later. + echo "Virtual environment detected. Adding packages: ${deps[*]}" + pip install --quiet --upgrade "${deps[@]}" + echo 'Starting the build.' + release_with python +else + function suggest_venv() { + venv_cmd='python -m venv env && source env/bin/activate' + printf "HELP: To avoid this error, use a virtual-env with '%s' instead.\n" "$venv_cmd" + } + trap suggest_venv ERR # This keeps the original exit (error) code. + echo 'Starting the build.' + release_with python3 # Outside a venv, use python3. +fi diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 03d387e..e6765cd 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -230,16 +230,16 @@ def store(self, istream): # end rename only if needed # Ensure rename is actually done and file is stable - # Retry up to 14 times - exponential wait & retry in ms. + # Retry up to 14 times - quadratic wait & retry in ms. # The total maximum wait time is 1000ms, which should be vastly enough for the # OS to return and commit the file to disk. - for exp_backoff_ms in [1, 4, 9, 16, 25, 36, 49, 64, 81, 100, 121, 144, 169, 181]: + for backoff_ms in [1, 4, 9, 16, 25, 36, 49, 64, 81, 100, 121, 144, 169, 181]: with suppress(PermissionError): # make sure its readable for all ! It started out as rw-- tmp file # but needs to be rwrr chmod(obj_path, self.new_objects_mode) break - time.sleep(exp_backoff_ms / 1000.0) + time.sleep(backoff_ms / 1000.0) else: raise PermissionError( "Impossible to apply `chmod` to file {}".format(obj_path) diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index bd30156..5536db0 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -2,7 +2,6 @@ # # This module is part of GitDB and is released under # the New BSD License: https://opensource.org/license/bsd-3-clause/ -import codecs from gitdb.db.base import ( CompoundDB, ) @@ -42,7 +41,7 @@ def _update_dbs_from_ref_file(self): # try to get as many as possible, don't fail if some are unavailable ref_paths = list() try: - with codecs.open(self._ref_file, 'r', encoding="utf-8") as f: + with open(self._ref_file, 'r', encoding="utf-8") as f: ref_paths = [l.strip() for l in f] except OSError: pass diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index f31bfa3..e4ad410 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit f31bfa378c8840d38d31e7e11ef2b84f191a491e +Subproject commit e4ad410ac3baf2046bd4043394e7cbb119045cc1 diff --git a/gitdb/pack.py b/gitdb/pack.py index e559e11..2e07079 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -267,8 +267,6 @@ def close(self): def _set_cache_(self, attr): if attr == "_packfile_checksum": self._packfile_checksum = self._cursor.map()[-40:-20] - elif attr == "_packfile_checksum": - self._packfile_checksum = self._cursor.map()[-20:] elif attr == "_cursor": # Note: We don't lock the file when reading as we cannot be sure # that we can actually write to the location - it could be a read-only @@ -848,7 +846,6 @@ def is_valid_stream(self, sha, use_crc=False): assert shawriter.sha(as_hex=False) == sha return shawriter.sha(as_hex=False) == sha # END handle crc/sha verification - return True def info_iter(self): """ diff --git a/gitdb/stream.py b/gitdb/stream.py index 1e0be84..7c59f0b 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -254,68 +254,93 @@ def read(self, size=-1): # copied once, and another copy of a part of it when it creates the unconsumed # tail. We have to use it to hand in the appropriate amount of bytes during # the next read. - tail = self._zip.unconsumed_tail - if tail: - # move the window, make it as large as size demands. For code-clarity, - # we just take the chunk from our map again instead of reusing the unconsumed - # tail. The latter one would safe some memory copying, but we could end up - # with not getting enough data uncompressed, so we had to sort that out as well. - # Now we just assume the worst case, hence the data is uncompressed and the window - # needs to be as large as the uncompressed bytes we want to read. - self._cws = self._cwe - len(tail) - self._cwe = self._cws + size - else: - cws = self._cws - self._cws = self._cwe - self._cwe = cws + size - # END handle tail - - # if window is too small, make it larger so zip can decompress something - if self._cwe - self._cws < 8: - self._cwe = self._cws + 8 - # END adjust winsize - - # takes a slice, but doesn't copy the data, it says ... - indata = self._m[self._cws:self._cwe] - - # get the actual window end to be sure we don't use it for computations - self._cwe = self._cws + len(indata) - dcompdat = self._zip.decompress(indata, size) - # update the amount of compressed bytes read - # We feed possibly overlapping chunks, which is why the unconsumed tail - # has to be taken into consideration, as well as the unused data - # if we hit the end of the stream - # NOTE: Behavior changed in PY2.7 onward, which requires special handling to make the tests work properly. - # They are thorough, and I assume it is truly working. - # Why is this logic as convoluted as it is ? Please look at the table in - # https://github.com/gitpython-developers/gitdb/issues/19 to learn about the test-results. - # Basically, on py2.6, you want to use branch 1, whereas on all other python version, the second branch - # will be the one that works. - # However, the zlib VERSIONs as well as the platform check is used to further match the entries in the - # table in the github issue. This is it ... it was the only way I could make this work everywhere. - # IT's CERTAINLY GOING TO BITE US IN THE FUTURE ... . - if getattr(zlib, 'ZLIB_RUNTIME_VERSION', zlib.ZLIB_VERSION) in ('1.2.7', '1.2.5') and not sys.platform == 'darwin': - unused_datalen = len(self._zip.unconsumed_tail) - else: - unused_datalen = len(self._zip.unconsumed_tail) + len(self._zip.unused_data) - # # end handle very special case ... - - self._cbr += len(indata) - unused_datalen - self._br += len(dcompdat) + # + # Decompress in a loop until we have produced `size` bytes or run out + # of progress. Iteration (instead of recursion) keeps the call bounded + # for streams that consume many input bytes per produced output byte + # (e.g. zlib stored blocks of length zero); the previous recursive + # form blew the stack on inputs > ~1500 empty blocks (issue #120 + # follow-up). + dcompdat = b'' + while True: + tail = self._zip.unconsumed_tail + remaining = size - len(dcompdat) + if tail: + # move the window, make it as large as size demands. For code-clarity, + # we just take the chunk from our map again instead of reusing the unconsumed + # tail. The latter one would save some memory copying, but we could end up + # with not getting enough data uncompressed, so we had to sort that out as well. + # Now we just assume the worst case, hence the data is uncompressed and the window + # needs to be as large as the uncompressed bytes we want to read. + self._cws = self._cwe - len(tail) + self._cwe = self._cws + remaining + else: + cws = self._cws + self._cws = self._cwe + self._cwe = cws + remaining + # END handle tail + + # if window is too small, make it larger so zip can decompress something + if self._cwe - self._cws < 8: + self._cwe = self._cws + 8 + # END adjust winsize + + # takes a slice, but doesn't copy the data, it says ... + indata = self._m[self._cws:self._cwe] + + # get the actual window end to be sure we don't use it for computations + self._cwe = self._cws + len(indata) + chunk = self._zip.decompress(indata, remaining) + # update the amount of compressed bytes read + # We feed possibly overlapping chunks, which is why the unconsumed tail + # has to be taken into consideration, as well as the unused data + # if we hit the end of the stream + # NOTE: Behavior changed in PY2.7 onward, which requires special handling to make the tests work properly. + # They are thorough, and I assume it is truly working. + # Why is this logic as convoluted as it is ? Please look at the table in + # https://github.com/gitpython-developers/gitdb/issues/19 to learn about the test-results. + # Basically, on py2.6, you want to use branch 1, whereas on all other python version, the second branch + # will be the one that works. + # However, the zlib VERSIONs as well as the platform check is used to further match the entries in the + # table in the github issue. This is it ... it was the only way I could make this work everywhere. + # IT's CERTAINLY GOING TO BITE US IN THE FUTURE ... . + if getattr(zlib, 'ZLIB_RUNTIME_VERSION', zlib.ZLIB_VERSION) in ('1.2.7', '1.2.5') and not sys.platform == 'darwin': + unused_datalen = len(self._zip.unconsumed_tail) + else: + unused_datalen = len(self._zip.unconsumed_tail) + len(self._zip.unused_data) + # # end handle very special case ... + + consumed = len(indata) - unused_datalen + self._cbr += consumed + self._br += len(chunk) + if chunk: + if not isinstance(dcompdat, bytearray): + dcompdat = bytearray(dcompdat) + dcompdat.extend(chunk) + + # Stop when we have enough or there is no path to more output. + # `chunk` may legitimately be empty mid-stream when zlib is + # consuming header / dictionary frames; in that case we keep + # iterating as long as we are still feeding zlib new bytes + # (consumed > 0) and zlib has not flagged end-of-stream. The + # compressed_bytes_read() scrub loop drives this same code with + # _br manipulated to 0 past zip EOF; it terminates here because + # `getattr(_zip, 'eof', False)` is True or no compressed bytes + # are consumed. The empty-block recursion attack from issue #120 + # follow-up is bounded by the iteration; each empty block does + # consume input, so the loop walks the stream forward a constant + # amount per iteration without growing the call stack. + if len(dcompdat) >= size or self._br >= self._s: + break + zip_eof = getattr(self._zip, 'eof', False) + if not chunk and (zip_eof or len(indata) == 0 or consumed == 0): + break + # END iterative decompress if dat: dcompdat = dat + dcompdat # END prepend our cached data - # it can happen, depending on the compression, that we get less bytes - # than ordered as it needs the final portion of the data as well. - # Recursively resolve that. - # Note: dcompdat can be empty even though we still appear to have bytes - # to read, if we are called by compressed_bytes_read - it manipulates - # us to empty the stream - if dcompdat and (len(dcompdat) - len(dat)) < size and self._br < self._s: - dcompdat += self.read(size - len(dcompdat)) - # END handle special case return dcompdat diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 1e7e941..f36b06b 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -162,3 +162,39 @@ def test_decompress_reader_special_case(self): dump = mdb.store(IStream(ostream.type, ostream.size, BytesIO(data))) assert dump.hexsha == sha # end for each loose object sha to test + + def test_decompress_reader_chunked_read_does_not_terminate_early(self): + """Regression test for #120: read(N) must not return b'' before EOF. + + zlib can consume input without producing decompressed output (e.g. + while ingesting block headers). The reader's internal recursion + previously bailed on any empty zip output, so a caller reading in + small chunks via the standard `while chunk := stream.read(N)` idiom + would terminate at the first empty chunk -- before the actual end + of the uncompressed stream. + """ + # Highly compressible data exposes the bug because each zlib chunk + # spans many uncompressed bytes -- intermediate decompress() calls + # often return empty while consuming input. + data = b"hello world! " * 1000 + zdata = zlib.compress(data) + + # Loop with a small chunk size to force many internal read/decompression + # iterations before EOF. + for chunk_size in (1, 4, 16, 64): + reader = DecompressMemMapReader( + zdata, close_on_deletion=False, size=len(data) + ) + out = bytearray() + while True: + chunk = reader.read(chunk_size) + if not chunk: + break + out.extend(chunk) + assert bytes(out) == data, ( + f"chunk_size={chunk_size}: got {len(out)}/{len(data)} bytes" + ) + assert reader._br == reader._s, ( + f"chunk_size={chunk_size}: stream stopped at " + f"{reader._br}/{reader._s}" + )