aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--repos/base.py35
1 files changed, 19 insertions, 16 deletions
diff --git a/repos/base.py b/repos/base.py
index 220a30f..1bd4ae3 100644
--- a/repos/base.py
+++ b/repos/base.py
@@ -1,9 +1,7 @@
from dataclasses import dataclass, asdict as dataclass_asdict
-import datetime
from functools import total_ordering
import gzip
import json
-import os
from pathlib import Path
import re
from typing import Any, Callable, Mapping
@@ -73,30 +71,35 @@ class Repository:
def get_versions(self) -> Mapping[str, Version]:
self._cache_dir().mkdir(parents=True, exist_ok=True)
+ headers = dict()
+
downloaded_file = self._cache_file('downloaded')
- if downloaded_file.exists():
- mtime = downloaded_file.stat().st_mtime
- else:
- mtime = 0
- mtime = datetime.datetime.fromtimestamp(mtime, datetime.timezone.utc)
- mtime = mtime.strftime(HTTP_DATE)
- parsed_file = self._cache_file('parsed.json.gz')
+ mtime_file = self._cache_file('last-modified')
+ if mtime_file.exists():
+ mtime = mtime_file.read_text()
+ headers['If-Modified-Since'] = mtime
- response = requests.get(self.index_url, headers={
- 'If-Modified-Since': mtime,
- }, stream=True)
+ etag_file = self._cache_file('etag')
+ if etag_file.exists():
+ etag = etag_file.read_text()
+ headers['If-None-Match'] = etag
+
+ response = requests.get(self.index_url, headers=headers, stream=True)
if response.status_code != requests.codes.not_modified:
response.raise_for_status()
print('Re-downloading', self._full_name())
- set_mtime = response.headers.get('Last-Modified', '')
with downloaded_file.open('wb') as f:
for chunk in response.iter_content(chunk_size=256):
f.write(chunk)
- if len(set_mtime) > 0:
- set_mtime = datetime.datetime.strptime(set_mtime, HTTP_DATE)
- os.utime(downloaded_file, (datetime.datetime.now().timestamp(), set_mtime.timestamp()))
+ if 'Last-Modified' in response.headers:
+ set_mtime = response.headers['Last-Modified']
+ mtime_file.write_text(set_mtime)
+ if 'ETag' in response.headers:
+ set_etag = response.headers['ETag']
+ etag_file.write_text(set_etag)
+ parsed_file = self._cache_file('parsed.json.gz')
if response.status_code != requests.codes.not_modified or not parsed_file.exists():
parsed_data = self.parse(downloaded_file)
with gzip.open(parsed_file, 'wt') as f: