colin/nix-files
1
0
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

feeds/update.py: consider https AND http as a fallback

Browse Source
This commit is contained in:
Colin 2023-03-08 09:19:36 +00:00
parent 8ae4be341a
commit 3c6da51f84
2 changed files with 8 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
pkgs/feeds/default.nix
View File

@ -28,14 +28,13 @@
'' ''
sources_dir=modules/data/feeds/sources sources_dir=modules/data/feeds/sources
name="$1" name="$1"
url="https://$name"
json_path="$sources_dir/$name/default.json" json_path="$sources_dir/$name/default.json"
# the name could have slashes in it, so we want to mkdir -p that # the name could have slashes in it, so we want to mkdir -p that
# but in a way where the least could go wrong. # but in a way where the least could go wrong.
pushd "$sources_dir"; mkdir -p "$name"; popd pushd "$sources_dir"; mkdir -p "$name"; popd
${./update.py} "$url" "$json_path" ${./update.py} "$name" "$json_path"
cat "$json_path" cat "$json_path"
''; '';
} }

10
pkgs/feeds/update.py
View File

@ -13,9 +13,13 @@ logging.getLogger().setLevel(logging.DEBUG)
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
logging.getLogger(__name__).debug("logging enabled") logging.getLogger(__name__).debug("logging enabled")
url = coerce_url(url, default_scheme="https") def try_scheme(url: str, scheme: str):
items = search(url, total_timeout=180, request_timeout=90, max_content_length=100*1024*1024) url = coerce_url(url, default_scheme=scheme)
items = sort_urls(items) print(f"trying {url}")
items = search(url, total_timeout=180, request_timeout=90, max_content_length=100*1024*1024)
return sort_urls(items)
items = try_scheme(url, "https") or try_scheme(url, "http")
# print all results # print all results
serialized = [item.serialize() for item in items] serialized = [item.serialize() for item in items]