diff options
| -rw-r--r-- | repos/base.py | 35 |
1 files changed, 19 insertions, 16 deletions
diff --git a/repos/base.py b/repos/base.py index 220a30f..1bd4ae3 100644 --- a/repos/base.py +++ b/repos/base.py @@ -1,9 +1,7 @@ from dataclasses import dataclass, asdict as dataclass_asdict -import datetime from functools import total_ordering import gzip import json -import os from pathlib import Path import re from typing import Any, Callable, Mapping @@ -73,30 +71,35 @@ class Repository: def get_versions(self) -> Mapping[str, Version]: self._cache_dir().mkdir(parents=True, exist_ok=True) + headers = dict() + downloaded_file = self._cache_file('downloaded') - if downloaded_file.exists(): - mtime = downloaded_file.stat().st_mtime - else: - mtime = 0 - mtime = datetime.datetime.fromtimestamp(mtime, datetime.timezone.utc) - mtime = mtime.strftime(HTTP_DATE) - parsed_file = self._cache_file('parsed.json.gz') + mtime_file = self._cache_file('last-modified') + if mtime_file.exists(): + mtime = mtime_file.read_text() + headers['If-Modified-Since'] = mtime - response = requests.get(self.index_url, headers={ - 'If-Modified-Since': mtime, - }, stream=True) + etag_file = self._cache_file('etag') + if etag_file.exists(): + etag = etag_file.read_text() + headers['If-None-Match'] = etag + + response = requests.get(self.index_url, headers=headers, stream=True) if response.status_code != requests.codes.not_modified: response.raise_for_status() print('Re-downloading', self._full_name()) - set_mtime = response.headers.get('Last-Modified', '') with downloaded_file.open('wb') as f: for chunk in response.iter_content(chunk_size=256): f.write(chunk) - if len(set_mtime) > 0: - set_mtime = datetime.datetime.strptime(set_mtime, HTTP_DATE) - os.utime(downloaded_file, (datetime.datetime.now().timestamp(), set_mtime.timestamp())) + if 'Last-Modified' in response.headers: + set_mtime = response.headers['Last-Modified'] + mtime_file.write_text(set_mtime) + if 'ETag' in response.headers: + set_etag = response.headers['ETag'] + etag_file.write_text(set_etag) + parsed_file = self._cache_file('parsed.json.gz') if response.status_code != requests.codes.not_modified or not parsed_file.exists(): parsed_data = self.parse(downloaded_file) with gzip.open(parsed_file, 'wt') as f: |