colin/nix-files
1
0
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

feeds/update.py: consider https AND http as a fallback

Browse Source
This commit is contained in:
Colin 2023-03-08 09:19:36 +00:00
parent 8ae4be341a
commit 3c6da51f84
2 changed files with 8 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
pkgs/feeds/default.nix
View File

@ -28,14 +28,13 @@
''
sources_dir=modules/data/feeds/sources
name="$1"
url="https://$name"
json_path="$sources_dir/$name/default.json"
# the name could have slashes in it, so we want to mkdir -p that
# but in a way where the least could go wrong.
pushd "$sources_dir"; mkdir -p "$name"; popd
${./update.py} "$url" "$json_path"
${./update.py} "$name" "$json_path"
cat "$json_path"
'';
}

10
pkgs/feeds/update.py
View File

@ -13,9 +13,13 @@ logging.getLogger().setLevel(logging.DEBUG)
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
logging.getLogger(__name__).debug("logging enabled")
url = coerce_url(url, default_scheme="https")
items = search(url, total_timeout=180, request_timeout=90, max_content_length=100*1024*1024)
items = sort_urls(items)
def try_scheme(url: str, scheme: str):
url = coerce_url(url, default_scheme=scheme)
print(f"trying {url}")
items = search(url, total_timeout=180, request_timeout=90, max_content_length=100*1024*1024)
return sort_urls(items)
items = try_scheme(url, "https") or try_scheme(url, "http")
# print all results
serialized = [item.serialize() for item in items]