From 8c0f231f734b5a9d736420eece311a87aa041103 Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Sat, 26 Feb 2022 12:29:39 +0100 Subject: README: example scripts to count new and unread items --- README | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) (limited to 'README') diff --git a/README b/README index 40037ab..398b96d 100644 --- a/README +++ b/README @@ -963,6 +963,58 @@ TSV format. - - - +Counting unread and total items +------------------------------- + +It can be useful to show the counts of unread items, for example in a +windowmanager or statusbar. + +The below example script counts the items of the last day in the same way the +formatting tools do: + + #!/bin/sh + # Count the new items of the last day. + LC_ALL=C awk -F '\t' -v "old=$(($(date +'%s') - 86400))" ' + { + total++; + } + int($1) >= old { + totalnew++; + } + END { + print "New: " totalnew; + print "Total: " total; + }' ~/.sfeed/urls ~/.sfeed/feeds/* + +The below example script counts the unread items using the sfeed_curses URL +file: + + #!/bin/sh + # Count the unread and total items from feeds using the URL file. + LC_ALL=C awk -F '\t' ' + # URL file: amount of fields is 1. + NF == 1 { + u[$0] = 1; # lookup table of URLs. + next; + } + # feed file: check by URL or id. + { + total++; + if (length($3)) { + if (u[$3]) + read++; + } else if (length($6)) { + if (u[$6]) + read++; + } + } + END { + print "Unread: " (total - read); + print "Total: " total; + }' ~/.sfeed/urls ~/.sfeed/feeds/* + +- - - + Running custom commands inside the program ------------------------------------------ -- cgit v1.2.3 From 489a3b12923ea6dc6a278bed023a40ab2b5375f0 Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Sat, 26 Feb 2022 12:37:38 +0100 Subject: README: small rewording --- README | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'README') diff --git a/README b/README index 398b96d..459f513 100644 --- a/README +++ b/README @@ -1015,8 +1015,8 @@ file: - - - -Running custom commands inside the program ------------------------------------------- +Running custom commands inside the sfeed_curses program +------------------------------------------------------- Running commands inside the sfeed_curses program can be useful for example to sync items or mark all items across all feeds as read. It can be comfortable to @@ -1035,13 +1035,12 @@ or forkexec((char *[]) { "syncnews.sh", NULL }, 1); break; -The specified script should be in $PATH or an absolute path. +The specified script should be in $PATH or be an absolute path. Example of a `markallread.sh` shellscript to mark all URLs as read: #!/bin/sh # mark all items/URLs as read. - tmp=$(mktemp) (cat ~/.sfeed/urls; cut -f 3 ~/.sfeed/feeds/*) | \ awk '!x[$0]++' > "$tmp" && -- cgit v1.2.3 From 26c14ee6dff96b1f7812070a17db161090ef87bd Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Sat, 5 Mar 2022 11:20:22 +0100 Subject: README: add a note about mouse button encoding issues in some terminals Maybe this is out-of-scope for the sfeed documentation though. --- README | 3 +++ 1 file changed, 3 insertions(+) (limited to 'README') diff --git a/README b/README index 459f513..40c5483 100644 --- a/README +++ b/README @@ -1081,6 +1081,9 @@ testing sfeed_curses. Some of them might be fixed already upstream: middle-button, right-button is incorrect / reversed. - putty: the full reset attribute (ESC c, typically `rs1`) does not reset the window title. +- Mouse button encoding for extended buttons (like side-buttons) in some + terminals are unsupported or map to the same button: for example side-buttons 7 + and 8 map to the scroll buttons 4 and 5 in urxvt. License -- cgit v1.2.3 From 4925af753dd44b935588b83c814301244a09bdd7 Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Mon, 7 Mar 2022 12:11:31 +0100 Subject: README: update codemadness.org URLs and use the full content feed by default --- README | 4 ++-- sfeedrc.5 | 4 ++-- sfeedrc.example | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) (limited to 'README') diff --git a/README b/README index 40c5483..a313cd8 100644 --- a/README +++ b/README @@ -245,8 +245,8 @@ Find RSS/Atom feed URLs from a webpage: output example: - https://codemadness.org/blog/rss.xml application/rss+xml - https://codemadness.org/blog/atom.xml application/atom+xml + https://codemadness.org/atom.xml application/atom+xml + https://codemadness.org/atom_content.xml application/atom+xml - - - diff --git a/sfeedrc.5 b/sfeedrc.5 index aedbf59..95ffc04 100644 --- a/sfeedrc.5 +++ b/sfeedrc.5 @@ -1,4 +1,4 @@ -.Dd August 5, 2021 +.Dd March 7, 2022 .Dt SFEEDRC 5 .Os .Sh NAME @@ -136,7 +136,7 @@ shown below: # list of feeds to fetch: feeds() { # feed [basesiteurl] [encoding] - feed "codemadness" "https://www.codemadness.nl/atom.xml" + feed "codemadness" "https://www.codemadness.org/atom_content.xml" feed "explosm" "http://feeds.feedburner.com/Explosm" feed "golang github releases" "https://github.com/golang/go/releases.atom" feed "linux kernel" "https://www.kernel.org/feeds/kdist.xml" "https://www.kernel.org" diff --git a/sfeedrc.example b/sfeedrc.example index 5ead13d..28630ec 100644 --- a/sfeedrc.example +++ b/sfeedrc.example @@ -3,7 +3,7 @@ # list of feeds to fetch: feeds() { # feed [basesiteurl] [encoding] - feed "codemadness" "https://www.codemadness.nl/atom.xml" + feed "codemadness" "https://www.codemadness.org/atom_content.xml" feed "explosm" "http://feeds.feedburner.com/Explosm" feed "golang github releases" "https://github.com/golang/go/releases.atom" feed "linux kernel" "https://www.kernel.org/feeds/kdist.xml" "https://www.kernel.org" -- cgit v1.2.3 From 373cabcf7b96ebafc8cd544de5f08bda3e81b375 Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Mon, 21 Mar 2022 11:01:46 +0100 Subject: util.h: slightly improve portability Tested with the scc compiler which is a pure c99 compiler. sys/types.h is not needed here anymore (it was used for ssize_t). Side-note: scc can now compile the sfeed parser program! It requires these changes at the time of writing: Add a strcasecmp and strncasecmp function and use getchar instead of getchar_unlocked. --- README | 2 +- util.h | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) (limited to 'README') diff --git a/README b/README index a313cd8..95f4e9a 100644 --- a/README +++ b/README @@ -153,7 +153,7 @@ OS tested --------- - Linux, - compilers: clang, gcc, chibicc, cproc, lacc, pcc, tcc, + compilers: clang, gcc, chibicc, cproc, lacc, pcc, scc, tcc, libc: glibc, musl. - OpenBSD (clang, gcc). - NetBSD (with NetBSD curses). diff --git a/util.h b/util.h index 8053750..7923cdc 100644 --- a/util.h +++ b/util.h @@ -1,6 +1,5 @@ -#include - #include +#include #ifdef __OpenBSD__ #include -- cgit v1.2.3 From a9445184bb7b4975bc27e7b41e7bfd9f31f1c0ac Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Mon, 21 Mar 2022 23:03:47 +0100 Subject: README: update sfeed_update_xargs example Set exit status non-zero if any of the feeds failed similar to what sfeed_update also does now. --- README | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'README') diff --git a/README b/README index 95f4e9a..648e63e 100644 --- a/README +++ b/README @@ -742,12 +742,18 @@ sfeed_update_xargs shellscript: # fetch feeds and store in temporary directory. sfeedtmpdir="$(mktemp -d '/tmp/sfeed_XXXXXX')" + mkdir -p "${sfeedtmpdir}/feeds" + touch "${sfeedtmpdir}/ok" # make sure path exists. mkdir -p "${sfeedpath}" # print feeds for parallel processing with xargs. feeds | SFEED_UPDATE_CHILD="1" xargs -r -0 -P "${maxjobs}" -L 6 "$(readlink -f "$0")" + status=$? + # check error exit status indicator for parallel jobs. + test -f "${sfeedtmpdir}/ok" || status=1 # cleanup temporary files etc. cleanup + exit ${status} - - - -- cgit v1.2.3 From da2726ca7789f7b6edf85cee496f20fdc1332afc Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Tue, 22 Mar 2022 10:58:25 +0100 Subject: README: improve error handling in downloader example - Return exit code, which makes xargs return 123 when any of the downloads failed. - Write errors to stderr. - Write non-errors to stdout. --- README | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'README') diff --git a/README b/README index 648e63e..db61a18 100644 --- a/README +++ b/README @@ -783,7 +783,7 @@ arguments are specified then the data is read from stdin. else s="$2" fi - printf '[%s]: %s: %s\n' "$(date +'%H:%M:%S')" "${s}" "$3" >&2 + printf '[%s]: %s: %s\n' "$(date +'%H:%M:%S')" "${s}" "$3" } # fetch(url, feedname) @@ -809,8 +809,8 @@ arguments are specified then the data is read from stdin. if [ "${feedname}" != "-" ]; then mkdir -p "${feedname}" if ! cd "${feedname}"; then - log "${feedname}" "${msg}: ${feedname}" "DIR FAIL" - exit 1 + log "${feedname}" "${msg}: ${feedname}" "DIR FAIL" >&2 + return 1 fi fi @@ -825,8 +825,10 @@ arguments are specified then the data is read from stdin. printf '%s\n' "${url}" >> "${cachefile}" ) 9>"${lockfile}" else - log "${feedname}" "${msg}" "FAIL" + log "${feedname}" "${msg}" "FAIL" >&2 + return 1 fi + return 0 } if [ "${SFEED_DOWNLOAD_CHILD}" = "1" ]; then -- cgit v1.2.3 From 70426c5b55dd9d1f96860644d868fb3399e5a4df Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Tue, 22 Mar 2022 17:29:11 +0100 Subject: README: sfeed_download: remove a line --- README | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'README') diff --git a/README b/README index db61a18..6ce4231 100644 --- a/README +++ b/README @@ -815,8 +815,7 @@ arguments are specified then the data is read from stdin. fi log "${feedname}" "${msg}" "START" - fetch "${url}" "${feedname}" - if [ $? = 0 ]; then + if fetch "${url}" "${feedname}"; then log "${feedname}" "${msg}" "OK" # append it safely in parallel to the cachefile on a -- cgit v1.2.3 From ca3f3fe68ae72fec6f607278bf88d30ab1497627 Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Fri, 25 Mar 2022 15:43:47 +0100 Subject: change echo to printf and for sfeed_opml_export use a control-character separator echo is unportable in this way and names containing characters like an option (-n) or backslash or escape codes (\e, \n, etc) could be messy. For awk set LC_ALL=C for simple collation. This makes sfeed_opml_export slower in some shells that don't have printf builtin though. For example with about 150 feeds in a config file it is a bit slower on OpenBSD ksh. time ./sfeed_opml_export | wc -l 152 0m00.29s real 0m00.05s user 0m00.20s system time sfeed_opml_export | wc -l 152 0m00.02s real 0m00.00s user 0m00.03s system --- README | 2 +- sfeed_markread | 2 +- sfeed_opml_export | 12 ++++++++---- sfeed_update | 4 ++-- 4 files changed, 12 insertions(+), 8 deletions(-) (limited to 'README') diff --git a/README b/README index 6ce4231..9658ab6 100644 --- a/README +++ b/README @@ -578,7 +578,7 @@ procmail_maildirs.sh file: mkdir -p "${maildir}/.cache" if ! test -r "${procmailconfig}"; then - echo "Procmail configuration file \"${procmailconfig}\" does not exist or is not readable." >&2 + printf "Procmail configuration file \"%s\" does not exist or is not readable.\n" "${procmailconfig}" >&2 echo "See procmailrc.example for an example." >&2 exit 1 fi diff --git a/sfeed_markread b/sfeed_markread index 4c509f3..b262bdd 100755 --- a/sfeed_markread +++ b/sfeed_markread @@ -2,7 +2,7 @@ # Mark items as read/unread: the input is the read / unread URL per line. usage() { - echo "usage: $0 [urlfile]" >&2 + printf "usage: %s [urlfile]\n" "$0" >&2 echo "" >&2 echo "An urlfile must be specified as an argument or with the environment variable \$SFEED_URL_FILE" >&2 exit 1 diff --git a/sfeed_opml_export b/sfeed_opml_export index eb22520..2a9396a 100755 --- a/sfeed_opml_export +++ b/sfeed_opml_export @@ -18,7 +18,7 @@ loadconfig() { if [ -r "${path}" ]; then . "${path}" else - echo "Configuration file \"${config}\" cannot be read." >&2 + printf "Configuration file \"%s\" cannot be read.\n" "${config}" >&2 echo "See sfeedrc.example for an example." >&2 exit 1 fi @@ -27,8 +27,8 @@ loadconfig() { # override feed function to output OPML XML. # feed(name, feedurl, [basesiteurl], [encoding]) feed() { - # TABs, newlines and echo options in field values are not checked. - echo "$1 $2" + # uses the characters 0x1f and 0x1e as a separator. + printf '%s\037%s\036' "$1" "$2" } # load config file. @@ -43,7 +43,11 @@ cat < ! -feeds | awk -F '\t' '{ +feeds | LC_ALL=C awk ' +BEGIN { + FS = "\x1f"; RS = "\x1e"; +} +{ gsub("&", "\\&"); gsub("\"", "\\""); gsub("'"'"'", "\\'"); diff --git a/sfeed_update b/sfeed_update index 2e54a59..857f537 100755 --- a/sfeed_update +++ b/sfeed_update @@ -27,7 +27,7 @@ loadconfig() { if [ -r "${path}" ]; then . "${path}" else - echo "Configuration file \"${config}\" cannot be read." >&2 + printf "Configuration file \"%s\" cannot be read.\n" "${config}" >&2 echo "See sfeedrc.example for an example." >&2 exit 1 fi @@ -187,7 +187,7 @@ sighandler() { } feeds() { - echo "Configuration file \"${config}\" is invalid or does not contain a \"feeds\" function." >&2 + printf "Configuration file \"%s\" is invalid or does not contain a \"feeds\" function.\n" "${config}" >&2 echo "See sfeedrc.example for an example." >&2 } -- cgit v1.2.3 From 2a47f88faae8e1e794c53ee7653d16dbe034bb15 Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Mon, 28 Mar 2022 12:55:34 +0200 Subject: update documentation of sfeed_update example and clarify return status Change the example to reload the feeds anyway, even if one of the feeds (temporarily) failed to update. --- README | 3 ++- sfeed_update.1 | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'README') diff --git a/README b/README index 9658ab6..86f30b2 100644 --- a/README +++ b/README @@ -1057,7 +1057,8 @@ Example of a `markallread.sh` shellscript to mark all URLs as read: Example of a `syncnews.sh` shellscript to update the feeds and reload them: #!/bin/sh - sfeed_update && pkill -SIGHUP sfeed_curses + sfeed_update + pkill -SIGHUP sfeed_curses Open an URL directly in the same terminal diff --git a/sfeed_update.1 b/sfeed_update.1 index 3edf551..893f290 100644 --- a/sfeed_update.1 +++ b/sfeed_update.1 @@ -1,4 +1,4 @@ -.Dd March 21, 2022 +.Dd March 28, 2022 .Dt SFEED_UPDATE 1 .Os .Sh NAME @@ -62,6 +62,7 @@ stdout in the format: .Ed .Sh EXIT STATUS .Ex -std +If any of the feeds failed to update then the return status is non-zero. .Sh EXAMPLES To update your feeds and format them in various formats: .Bd -literal -- cgit v1.2.3 From f4cfa079c74b2dfdf99ffa987fecf23b2f0f4f14 Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Fri, 1 Apr 2022 00:57:34 +0200 Subject: README: reflect example with the description --- README | 1 + 1 file changed, 1 insertion(+) (limited to 'README') diff --git a/README b/README index 86f30b2..92fe929 100644 --- a/README +++ b/README @@ -76,6 +76,7 @@ HTML view (no frames), copy style.css for a default style: HTML view with the menu as frames, copy style.css for a default style: mkdir -p "$HOME/.sfeed/frames" + cp style.css "$HOME/.sfeed/frames/style.css" cd "$HOME/.sfeed/frames" && sfeed_frames $HOME/.sfeed/feeds/* To automatically update your feeds periodically and format them in a way you -- cgit v1.2.3 From e1676198f01a1ba713e1bc9705db8df80cc47cfa Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Sat, 14 May 2022 12:28:58 +0200 Subject: README: add progress indicator script --- README | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) (limited to 'README') diff --git a/README b/README index 92fe929..92d77fe 100644 --- a/README +++ b/README @@ -971,6 +971,53 @@ TSV format. - - - +Progress indicator +------------------ + +The below sfeed_update wrapper script counts the amount of feeds in a sfeedrc +config. It then calls sfeed_update and pipes the output lines to a function +that counts the current progress. It writes the total progress to stderr. +Alternative: pv -l -s totallines + + #!/bin/sh + # Progress indicator script. + + # Pass lines as input to stdin and write progress status to stderr. + # progress(totallines) + progress() { + total="$(($1 + 0))" # must be a number, no divide by zero. + test "${total}" -le 0 -o "$1" != "${total}" && return + LC_ALL=C awk -v "total=${total}" ' + { + counter++; + percent = (counter * 100) / total; + printf("\033[K") > "/dev/stderr"; # clear EOL + print $0; + printf("[%s/%s] %.0f%%\r", counter, total, percent) > "/dev/stderr"; + fflush(); # flush all buffers per line. + } + END { + printf("\033[K") > "/dev/stderr"; + }' + } + + # Counts the feeds from the sfeedrc config. + countfeeds() { + count=0 + . "$1" + feed() { + count=$((count + 1)) + } + feeds + echo "${count}" + } + + config="${1:-$HOME/.sfeed/sfeedrc}" + total=$(countfeeds "${config}") + sfeed_update "${config}" | progress "${total}" + +- - - + Counting unread and total items ------------------------------- -- cgit v1.2.3 From 0391bae406896b068c9e82b76aa6c26c7928efc7 Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Thu, 19 May 2022 21:32:42 +0200 Subject: README: make sure sfeed_update errors don't write to stderr It would mess up the output since the progress bar writes to stderr aswell. --- README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'README') diff --git a/README b/README index 92d77fe..7753a8e 100644 --- a/README +++ b/README @@ -1014,7 +1014,7 @@ Alternative: pv -l -s totallines config="${1:-$HOME/.sfeed/sfeedrc}" total=$(countfeeds "${config}") - sfeed_update "${config}" | progress "${total}" + sfeed_update "${config}" 2>&1 | progress "${total}" - - - -- cgit v1.2.3 From f8ef20b2593304fd08a4599edfe3a66aa3a845e3 Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Tue, 31 May 2022 11:33:06 +0200 Subject: README: fix bug in an example for counting the new items of the last day This should not use the plain-text URL file ($SFEED_URL_FILE). --- README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'README') diff --git a/README b/README index 7753a8e..ece2f28 100644 --- a/README +++ b/README @@ -1039,7 +1039,7 @@ formatting tools do: END { print "New: " totalnew; print "Total: " total; - }' ~/.sfeed/urls ~/.sfeed/feeds/* + }' ~/.sfeed/feeds/* The below example script counts the unread items using the sfeed_curses URL file: -- cgit v1.2.3 From 0f2ddea7cfe31a651533cf170bc31627415db593 Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Sat, 18 Jun 2022 22:52:44 +0200 Subject: README: add setsid example --- README | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'README') diff --git a/README b/README index ece2f28..06f2c02 100644 --- a/README +++ b/README @@ -1109,6 +1109,21 @@ Example of a `syncnews.sh` shellscript to update the feeds and reload them: pkill -SIGHUP sfeed_curses +Running programs in a new session +--------------------------------- + +By default processes are spawned in the same session and process group as +sfeed_curses. When sfeed_curses is closed this can also close the spawned +process in some cases. + +When the setsid command-line program is available the following wrapper command +can be used to run the program in a new session, for a plumb program: + + setsid -f xdg-open "$@" + +Alternatively the code can be changed to call setsid() before execvp(). + + Open an URL directly in the same terminal ----------------------------------------- -- cgit v1.2.3 From 70007b61aaa485c7084badc5d8336ee480b2d138 Mon Sep 17 00:00:00 2001 From: NRK Date: Tue, 21 Jun 2022 05:09:14 +0600 Subject: fix some typis found via codespell $ codespell --ignore-regex Nd --- README | 2 +- README.xml | 4 ++-- sfeed_update | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) (limited to 'README') diff --git a/README b/README index 06f2c02..b994c55 100644 --- a/README +++ b/README @@ -834,7 +834,7 @@ arguments are specified then the data is read from stdin. if [ "${SFEED_DOWNLOAD_CHILD}" = "1" ]; then # Downloader helper for parallel downloading. # Receives arguments: $1 = URL, $2 = title, $3 = feed filename or "-". - # It should write the URI to the cachefile if it is succesful. + # It should write the URI to the cachefile if it is successful. downloader "$1" "$2" "$3" exit $? fi diff --git a/README.xml b/README.xml index edbac28..ee205c0 100644 --- a/README.xml +++ b/README.xml @@ -28,7 +28,7 @@ Supports - Tags in short-form (). - Tag attributes. -- Short attributes without an explicity set value (). +- Short attributes without an explicitly set value (). - Comments - CDATA sections. - Helper function (xml_entitytostr) to convert XML 1.0 / HTML 2.0 named entities @@ -55,7 +55,7 @@ Caveats - The XML specification has no limits on tag and attribute names. For simplicity/sanity sake this XML parser takes some liberties. Tag and attribute names are truncated if they are excessively long. -- Entity expansions are not parsed aswell as DOCTYPE, ATTLIST etc. +- Entity expansions are not parsed as well as DOCTYPE, ATTLIST etc. Files used diff --git a/sfeed_update b/sfeed_update index fc7447f..44ce23c 100755 --- a/sfeed_update +++ b/sfeed_update @@ -182,7 +182,7 @@ sighandler() { signo="$1" # ignore TERM signal for myself. trap -- "" TERM - # kill all running childs >:D + # kill all running children >:D kill -TERM -$$ } -- cgit v1.2.3 From 7d4043084c2e933603243f259f4f9f080e1d8911 Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Wed, 28 Dec 2022 11:18:44 +0100 Subject: README: small rewording "can be supported" -> "are supported" --- README | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'README') diff --git a/README b/README index b994c55..a4c2d25 100644 --- a/README +++ b/README @@ -146,8 +146,8 @@ sfeed supports a subset of XML 1.0 and a subset of: - MediaRSS extensions (media:). - Dublin Core extensions (dc:). -Other formats like JSONfeed, twtxt or certain RSS/Atom extensions can be -supported by converting them to RSS/Atom or to the sfeed(5) format directly. +Other formats like JSONfeed, twtxt or certain RSS/Atom extensions are supported +by converting them to RSS/Atom or to the sfeed(5) format directly. OS tested -- cgit v1.2.3 From aeb1398411ce245fa7982365640f7852d63b3d52 Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Sat, 4 Feb 2023 12:34:56 +0100 Subject: README: describe how to add new parsed tags and fields to sfeed.c --- README | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) (limited to 'README') diff --git a/README b/README index a4c2d25..fba7565 100644 --- a/README +++ b/README @@ -1070,6 +1070,96 @@ file: - - - +sfeed.c: adding new XML tags or sfeed(5) fields to the parser +------------------------------------------------------------- + +sfeed.c contains definitions to parse XML tags and map them to sfeed(5) TSV +fields. Parsed RSS and Atom tag names are first stored as a TagId, which is a +number. This TagId is then mapped to the output field index. + +* Add a new TagId enum for the tag. + +* (optional) Add a new FeedField* enum for the new output field or you can map + it to an existing field. + +* Add the new XML tag name to the array variable of parsed RSS or Atom + tags: rsstags[] or atomtags[]. + + These must be defined in alphabetical order, because a binary search is used + which uses the strcasecmp() function. + +* Add the parsed TagId to the output field in the array variable fieldmap[]. + + When another tag is also mapped to the same output field then the tag with + the highest TagId number value overrides the mapped field: the order is from + least important to high. + +* If this defined tag is just using the inner data of the XML tag, then this + definition is enough. If it for example has to parse a certain attribute you + have to add a check for the TagId to the xmlattr() callback function. + +* (optional) Print the new field in the printfields() function. + +Below is a patch example to add the MRSS "media:content" field as a new field: + +diff --git a/sfeed.c b/sfeed.c +--- a/sfeed.c ++++ b/sfeed.c +@@ -50,7 +50,7 @@ enum TagId { + RSSTagGuidPermalinkTrue, + /* must be defined after GUID, because it can be a link (isPermaLink) */ + RSSTagLink, +- RSSTagEnclosure, ++ RSSTagMediaContent, RSSTagEnclosure, + RSSTagAuthor, RSSTagDccreator, + RSSTagCategory, + /* Atom */ +@@ -81,7 +81,7 @@ typedef struct field { + enum { + FeedFieldTime = 0, FeedFieldTitle, FeedFieldLink, FeedFieldContent, + FeedFieldId, FeedFieldAuthor, FeedFieldEnclosure, FeedFieldCategory, +- FeedFieldLast ++ FeedFieldMediaContent, FeedFieldLast + }; + + typedef struct feedcontext { +@@ -137,6 +137,7 @@ static const FeedTag rsstags[] = { + { STRP("enclosure"), RSSTagEnclosure }, + { STRP("guid"), RSSTagGuid }, + { STRP("link"), RSSTagLink }, ++ { STRP("media:content"), RSSTagMediaContent }, + { STRP("media:description"), RSSTagMediaDescription }, + { STRP("pubdate"), RSSTagPubdate }, + { STRP("title"), RSSTagTitle } +@@ -180,6 +181,7 @@ static const int fieldmap[TagLast] = { + [RSSTagGuidPermalinkFalse] = FeedFieldId, + [RSSTagGuidPermalinkTrue] = FeedFieldId, /* special-case: both a link and an id */ + [RSSTagLink] = FeedFieldLink, ++ [RSSTagMediaContent] = FeedFieldMediaContent, + [RSSTagEnclosure] = FeedFieldEnclosure, + [RSSTagAuthor] = FeedFieldAuthor, + [RSSTagDccreator] = FeedFieldAuthor, +@@ -677,6 +679,8 @@ printfields(void) + string_print_uri(&ctx.fields[FeedFieldEnclosure].str); + putchar(FieldSeparator); + string_print_trimmed_multi(&ctx.fields[FeedFieldCategory].str); ++ putchar(FieldSeparator); ++ string_print_trimmed(&ctx.fields[FeedFieldMediaContent].str); + putchar('\n'); + + if (ferror(stdout)) /* check for errors but do not flush */ +@@ -718,7 +722,7 @@ xmlattr(XMLParser *p, const char *t, size_t tl, const char *n, size_t nl, + } + + if (ctx.feedtype == FeedTypeRSS) { +- if (ctx.tag.id == RSSTagEnclosure && ++ if ((ctx.tag.id == RSSTagEnclosure || ctx.tag.id == RSSTagMediaContent) && + isattr(n, nl, STRP("url"))) { + string_append(&tmpstr, v, vl); + } else if (ctx.tag.id == RSSTagGuid && + +- - - + Running custom commands inside the sfeed_curses program ------------------------------------------------------- -- cgit v1.2.3 From 3ee28da0f47d6a3b59d1c8493374df4e01bc0af4 Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Thu, 16 Feb 2023 20:29:22 +0100 Subject: README: small rewording in how to add new parsed tags and fields --- README | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'README') diff --git a/README b/README index fba7565..26de06c 100644 --- a/README +++ b/README @@ -1077,6 +1077,8 @@ sfeed.c contains definitions to parse XML tags and map them to sfeed(5) TSV fields. Parsed RSS and Atom tag names are first stored as a TagId, which is a number. This TagId is then mapped to the output field index. +Steps to modify the code: + * Add a new TagId enum for the tag. * (optional) Add a new FeedField* enum for the new output field or you can map @@ -1100,7 +1102,7 @@ number. This TagId is then mapped to the output field index. * (optional) Print the new field in the printfields() function. -Below is a patch example to add the MRSS "media:content" field as a new field: +Below is a patch example to add the MRSS "media:content" tag as a new field: diff --git a/sfeed.c b/sfeed.c --- a/sfeed.c -- cgit v1.2.3 From 9d6dd2a5072c0702cb56cced861ca4539e98c553 Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Fri, 17 Feb 2023 15:39:29 +0100 Subject: sfeed_update, sfeed_opml_export, README: reference the example sfeedrc man page ... and some small rewording. --- README | 4 ++-- sfeed_opml_export | 2 +- sfeed_update | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) (limited to 'README') diff --git a/README b/README index 26de06c..dcff323 100644 --- a/README +++ b/README @@ -251,8 +251,8 @@ output example: - - - -Make sure your sfeedrc config file exists, see sfeedrc.example. To update your -feeds (configfile argument is optional): +Make sure your sfeedrc config file exists, see the sfeedrc.example file. To +update your feeds (configfile argument is optional): sfeed_update "configfile" diff --git a/sfeed_opml_export b/sfeed_opml_export index 2a9396a..3835934 100755 --- a/sfeed_opml_export +++ b/sfeed_opml_export @@ -19,7 +19,7 @@ loadconfig() { . "${path}" else printf "Configuration file \"%s\" cannot be read.\n" "${config}" >&2 - echo "See sfeedrc.example for an example." >&2 + echo "See the sfeedrc.example file or the sfeedrc(5) man page for an example." >&2 exit 1 fi } diff --git a/sfeed_update b/sfeed_update index 44ce23c..f989be6 100755 --- a/sfeed_update +++ b/sfeed_update @@ -28,7 +28,7 @@ loadconfig() { . "${path}" else printf "Configuration file \"%s\" cannot be read.\n" "${config}" >&2 - echo "See sfeedrc.example for an example." >&2 + echo "See the sfeedrc.example file or the sfeedrc(5) man page for an example." >&2 exit 1 fi } -- cgit v1.2.3 From 04db56ad05b207374c973b8651a6bfa14bbbac4b Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Sun, 26 Mar 2023 15:30:18 +0200 Subject: README: add Open Watcom as a tested compiler Tested sfeed, sfeed_plain and sfeed_html for now. There are minor changes required for sfeed and some additional compatibility function changes required to make the format tools compile. Rough list of changes required: - xml.c: rename getchar_unlocked -> getchar - sfeed.c: EOVERFLOW -> ENOMEM - wcwidth: just filter ASCII range < 32 = -1 and > 127 = 0, rest = 1. - getline - localtime_r -> just do a localtime + memcpy. Make sure to increase stack size when compiling: owcc -k32768 -Os -s -o sfeed.exe sfeed.c util.c xml.c Resulting .exe binary is 27KB, 17KB compressed with UPX. The HTML output looks fine in Netscape Navigator and Internet Explorer 3 :) --- README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'README') diff --git a/README b/README index dcff323..35ecd75 100644 --- a/README +++ b/README @@ -165,7 +165,7 @@ OS tested - Windows (cygwin gcc + mintty, mingw). - HaikuOS - SerenityOS -- FreeDOS (djgpp). +- FreeDOS (djgpp, Open Watcom). - FUZIX (sdcc -mz80, with the sfeed parser program). -- cgit v1.2.3 From 15d18283616a951f060daf8f96996def532b8e21 Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Thu, 6 Apr 2023 19:35:24 +0200 Subject: README: fix typo JSONfeed -> JSON Feed https://www.jsonfeed.org/version/1.1/ --- README | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'README') diff --git a/README b/README index 35ecd75..6611793 100644 --- a/README +++ b/README @@ -146,8 +146,8 @@ sfeed supports a subset of XML 1.0 and a subset of: - MediaRSS extensions (media:). - Dublin Core extensions (dc:). -Other formats like JSONfeed, twtxt or certain RSS/Atom extensions are supported -by converting them to RSS/Atom or to the sfeed(5) format directly. +Other formats like JSON Feed, twtxt or certain RSS/Atom extensions are +supported by converting them to RSS/Atom or to the sfeed(5) format directly. OS tested -- cgit v1.2.3 From f11a83207e85e9007cf908a50318d50ef1e7bab9 Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Thu, 6 Apr 2023 20:22:00 +0200 Subject: README: improve error checking for temporary files in examples Similar to the recent fix for sfeed_update in commit 5a27c58675ddf4113d64a84f715cb3fecb681a6d --- README | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'README') diff --git a/README b/README index 6611793..3cb650d 100644 --- a/README +++ b/README @@ -742,9 +742,9 @@ sfeed_update_xargs shellscript: } # fetch feeds and store in temporary directory. - sfeedtmpdir="$(mktemp -d '/tmp/sfeed_XXXXXX')" + sfeedtmpdir="$(mktemp -d '/tmp/sfeed_XXXXXX')" || exit 1 mkdir -p "${sfeedtmpdir}/feeds" - touch "${sfeedtmpdir}/ok" + touch "${sfeedtmpdir}/ok" || exit 1 # make sure path exists. mkdir -p "${sfeedpath}" # print feeds for parallel processing with xargs. @@ -841,7 +841,7 @@ arguments are specified then the data is read from stdin. # ...else parent mode: - tmp=$(mktemp) + tmp="$(mktemp)" || exit 1 trap "rm -f ${tmp}" EXIT [ -f "${cachefile}" ] || touch "${cachefile}" @@ -1188,7 +1188,7 @@ Example of a `markallread.sh` shellscript to mark all URLs as read: #!/bin/sh # mark all items/URLs as read. - tmp=$(mktemp) + tmp="$(mktemp)" || exit 1 (cat ~/.sfeed/urls; cut -f 3 ~/.sfeed/feeds/*) | \ awk '!x[$0]++' > "$tmp" && mv "$tmp" ~/.sfeed/urls && -- cgit v1.2.3 From 0c938df27bf335fb290d6ea3612f5b397ed3da56 Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Thu, 6 Apr 2023 20:22:54 +0200 Subject: README: sfeed_download: change youtube-dl to yt-dlp This is an active maintained fork. --- README | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'README') diff --git a/README b/README index 3cb650d..e633a80 100644 --- a/README +++ b/README @@ -771,7 +771,7 @@ arguments are specified then the data is read from stdin. #!/bin/sh # sfeed_download: downloader for URLs and enclosures in sfeed(5) files. - # Dependencies: awk, curl, flock, xargs (-P), youtube-dl. + # Dependencies: awk, curl, flock, xargs (-P), yt-dlp. cachefile="${SFEED_CACHEFILE:-$HOME/.sfeed/downloaded_urls}" jobs="${SFEED_JOBS:-4}" @@ -791,7 +791,7 @@ arguments are specified then the data is read from stdin. fetch() { case "$1" in *youtube.com*) - youtube-dl "$1";; + yt-dlp "$1";; *.flac|*.ogg|*.m3u|*.m3u8|*.m4a|*.mkv|*.mp3|*.mp4|*.wav|*.webm) # allow 2 redirects, hide User-Agent, connect timeout is 15 seconds. curl -O -L --max-redirs 2 -H "User-Agent:" -f -s --connect-timeout 15 "$1";; -- cgit v1.2.3 From ced0dd7f8a01dedad3ba16c4cf209aea673c3e82 Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Mon, 10 Apr 2023 17:03:57 +0200 Subject: sfeed_json: add JSON output format tool This outputs the TSV data to JSON. It uses a subset of JSON Feed 1.1: https://www.jsonfeed.org/version/1.1/ --- Makefile | 1 + README | 1 + sfeed_json.1 | 48 +++++++++++++++++ sfeed_json.c | 171 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 221 insertions(+) create mode 100644 sfeed_json.1 create mode 100644 sfeed_json.c (limited to 'README') diff --git a/Makefile b/Makefile index c167614..a8493ba 100644 --- a/Makefile +++ b/Makefile @@ -51,6 +51,7 @@ BIN = \ sfeed_frames\ sfeed_gopher\ sfeed_html\ + sfeed_json\ sfeed_mbox\ sfeed_opml_import\ sfeed_plain\ diff --git a/README b/README index e633a80..a0a4815 100644 --- a/README +++ b/README @@ -186,6 +186,7 @@ sfeed_curses - Format feed data (TSV) to a curses interface. sfeed_frames - Format feed data (TSV) to HTML file(s) with frames. sfeed_gopher - Format feed data (TSV) to Gopher files. sfeed_html - Format feed data (TSV) to HTML. +sfeed_json - Format feed data (TSV) to JSON Feed. sfeed_opml_export - Generate an OPML XML file from a sfeedrc config file. sfeed_opml_import - Generate a sfeedrc config file from an OPML XML file. sfeed_markread - Mark items as read/unread, for use with sfeed_curses. diff --git a/sfeed_json.1 b/sfeed_json.1 new file mode 100644 index 0000000..8bd6ab0 --- /dev/null +++ b/sfeed_json.1 @@ -0,0 +1,48 @@ +.Dd March 8, 2023 +.Dt SFEED_JSON 1 +.Os +.Sh NAME +.Nm sfeed_json +.Nd format feed data to JSON Feed +.Sh SYNOPSIS +.Nm +.Op Ar +.Sh DESCRIPTION +.Nm +formats feed data (TSV) from +.Xr sfeed 1 +from stdin or for each +.Ar file +to stdout as JSON Feed data. +If one or more +.Ar file +arguments are specified then the basename of the +.Ar file +is used as the feed name in the output. +If no +.Ar file +arguments are specified and so the data is read from stdin then the feed name +is empty. +If +.Nm +is reading from one or more +.Ar file +arguments it will prefix the entry title with "[feed name] ". +.Sh EXIT STATUS +.Ex -std +.Sh EXAMPLES +.Bd -literal +curl -s 'https://codemadness.org/atom.xml' | sfeed | sfeed_json +.Ed +.Sh SEE ALSO +.Xr sfeed 1 , +.Xr sfeed_atom 1 , +.Xr sfeed 5 +.Sh STANDARDS +.Rs +.%T JSON Feed Version 1.1 +.%U https://www.jsonfeed.org/version/1.1/ +.%D Nov, 2022 +.Re +.Sh AUTHORS +.An Hiltjo Posthuma Aq Mt hiltjo@codemadness.org diff --git a/sfeed_json.c b/sfeed_json.c new file mode 100644 index 0000000..f6bb904 --- /dev/null +++ b/sfeed_json.c @@ -0,0 +1,171 @@ +#include +#include +#include +#include + +#include "util.h" + +static char *line; +static size_t linesize; +static int firstitem = 1; + +/* Unescape / decode fields printed by string_print_encoded() */ +static void +printcontent(const char *s) +{ + for (; *s; s++) { + switch (*s) { + case '\\': + s++; + switch (*s) { + case 'n': fputs("\\n", stdout); break; + case '\\': fputs("\\\\", stdout); break; + case 't': fputs("\\t", stdout); break; + } + break; /* ignore invalid escape sequence */ + case '"': fputs("\\\"", stdout); break; + default: + putchar(*s); + break; + } + } +} + +static void +printfield(const char *s) +{ + for (; *s; s++) { + if (*s == '\\') + fputs("\\\\", stdout); + else if (*s == '"') + fputs("\\\"", stdout); + else + putchar(*s); + } +} + +static void +printfeed(FILE *fp, const char *feedname) +{ + char *fields[FieldLast], timebuf[32]; + struct tm parsedtm, *tm; + time_t parsedtime; + ssize_t linelen; + int ch; + char *p, *s; + + while ((linelen = getline(&line, &linesize, fp)) > 0 && + !ferror(stdout)) { + if (line[linelen - 1] == '\n') + line[--linelen] = '\0'; + parseline(line, fields); + + if (!firstitem) + fputs(",\n", stdout); + firstitem = 0; + + fputs("{\n\t\"id\": \"", stdout); + printfield(fields[FieldId]); + fputs("\"", stdout); + + parsedtime = 0; + if (!strtotime(fields[FieldUnixTimestamp], &parsedtime) && + (tm = gmtime_r(&parsedtime, &parsedtm)) && + strftime(timebuf, sizeof(timebuf), "%Y-%m-%dT%H:%M:%SZ", tm)) { + fputs(",\n\t\"date_published\": \"", stdout); + fputs(timebuf, stdout); + fputs("\"", stdout); + } + + fputs(",\n\t\"title\": \"", stdout); + if (feedname[0]) { + fputs("[", stdout); + printfield(feedname); + fputs("] ", stdout); + } + printfield(fields[FieldTitle]); + fputs("\"", stdout); + + if (fields[FieldLink][0]) { + fputs(",\n\t\"url\": \"", stdout); + printfield(fields[FieldLink]); + fputs("\"", stdout); + } + + if (fields[FieldAuthor][0]) { + fputs(",\n\t\"authors\": [{\"name\": \"", stdout); + printfield(fields[FieldAuthor]); + fputs("\"}]", stdout); + } + + if (fields[FieldCategory][0]) { + fputs(",\n\t\"tags\": [", stdout); + + for (p = s = fields[FieldCategory]; ; s++) { + if (*s == '|' || *s == '\0') { + if (p != fields[FieldCategory]) + fputs(", ", stdout); + ch = *s; + *s = '\0'; /* temporary NUL terminate */ + fputs("\"", stdout); + printfield(p); + fputs("\"", stdout); + *s = ch; /* restore */ + p = s + 1; + } + if (*s == '\0') + break; + } + fputs("]", stdout); + } + + if (fields[FieldEnclosure][0]) { + fputs(",\n\t\"attachments\": [{\"url:\": \"", stdout); + printfield(fields[FieldEnclosure]); + fputs("\"}]", stdout); + } + + if (!strcmp(fields[FieldContentType], "html")) + fputs(",\n\t\"content_html\": \"", stdout); + else + fputs(",\n\t\"content_text\": \"", stdout); + printcontent(fields[FieldContent]); + fputs("\"\n}", stdout); + } +} + +int +main(int argc, char *argv[]) +{ + FILE *fp; + char *name; + int i; + + if (pledge(argc == 1 ? "stdio" : "stdio rpath", NULL) == -1) + err(1, "pledge"); + + fputs("{\n" + "\"version\": \"https://jsonfeed.org/version/1.1\",\n" + "\"title\": \"Newsfeed\",\n" + "\"items\": [\n", stdout); + + if (argc == 1) { + printfeed(stdin, ""); + checkfileerror(stdin, "", 'r'); + } else { + for (i = 1; i < argc; i++) { + if (!(fp = fopen(argv[i], "r"))) + err(1, "fopen: %s", argv[i]); + name = ((name = strrchr(argv[i], '/'))) ? name + 1 : argv[i]; + printfeed(fp, name); + checkfileerror(fp, argv[i], 'r'); + checkfileerror(stdout, "", 'w'); + fclose(fp); + } + } + fputs("]\n}\n", stdout); + + checkfileerror(stdout, "", 'w'); + + return 0; +} -- cgit v1.2.3 From 728270f69c34a84cb10aa891178c90c8fe36320d Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Wed, 12 Apr 2023 22:55:54 +0200 Subject: fix some typos --- README | 2 +- sfeed.c | 10 +++++----- sfeed_gopher.c | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) (limited to 'README') diff --git a/README b/README index a0a4815..6892e89 100644 --- a/README +++ b/README @@ -38,7 +38,7 @@ Initial setup: cp sfeedrc.example "$HOME/.sfeed/sfeedrc" Edit the sfeedrc(5) configuration file and change any RSS/Atom feeds. This file -is included and evaluated as a shellscript for sfeed_update, so it's functions +is included and evaluated as a shellscript for sfeed_update, so its functions and behaviour can be overridden: $EDITOR "$HOME/.sfeed/sfeedrc" diff --git a/sfeed.c b/sfeed.c index e96ffa3..e2f44b8 100644 --- a/sfeed.c +++ b/sfeed.c @@ -383,7 +383,7 @@ string_print_trimmed_multi(String *s) } } -/* Print URL, if it's a relative URL then it uses the global `baseurl`. */ +/* Print URL, if it is a relative URL then it uses the global `baseurl`. */ static void printuri(char *s) { @@ -409,7 +409,7 @@ printuri(char *s) *e = c; /* restore NUL byte to original character */ } -/* Print URL, if it's a relative URL then it uses the global `baseurl`. */ +/* Print URL, if it is a relative URL then it uses the global `baseurl`. */ static void string_print_uri(String *s) { @@ -432,7 +432,7 @@ string_print_timestamp(String *s) printf("%lld", t); } -/* Convert time fields. Returns a signed (atleast) 64-bit UNIX timestamp. +/* Convert time fields. Returns a signed (at least) 64-bit UNIX timestamp. Parameters should be passed as they are in a struct tm: that is: year = year - 1900, month = month - 1. */ static long long @@ -907,7 +907,7 @@ xmltagstartparsed(XMLParser *p, const char *t, size_t tl, int isshort) return; } - /* set tag type based on it's attribute value */ + /* set tag type based on its attribute value */ if (ctx.tag.id == RSSTagGuid) { /* if empty the default is "true" */ if (!attrispermalink.len || @@ -1018,7 +1018,7 @@ xmltagend(XMLParser *p, const char *t, size_t tl, int isshort) } /* temporary string: for fields that cannot be processed - directly and need more context, for example by it's tag + directly and need more context, for example by its tag attributes, like the Atom link rel="alternate|enclosure". */ if (tmpstr.len && ctx.field) { if (ISFEEDFIELDMULTI(fieldmap[ctx.tag.id])) { diff --git a/sfeed_gopher.c b/sfeed_gopher.c index 290d2d2..f62c6ed 100644 --- a/sfeed_gopher.c +++ b/sfeed_gopher.c @@ -64,7 +64,7 @@ printfeed(FILE *fpitems, FILE *fpin, struct feed *f) if (fields[FieldLink][0]) { itemtype = 'h'; - /* if it's a gopher URL then change it into a DirEntity */ + /* if it is a gopher URL then change it into a DirEntity */ if (!strncmp(fields[FieldLink], "gopher://", 9) && uri_parse(fields[FieldLink], &u) != -1) { itemhost = u.host; -- cgit v1.2.3 From 6e4136753bd0faa15c198118b05c529d96d908c5 Mon Sep 17 00:00:00 2001 From: Alvar Penning Date: Mon, 24 Apr 2023 22:01:23 +0200 Subject: sfeed_{curses,frames,gopher,html,plain}: SFEED_NEW_MAX_SECS By introducing the new environment variable $SFEED_NEW_MAX_SECS in all sfeed_* utilities marking feeds as new based on comparing their age, it is now possible to override this age limit. This allows, for example, to be notified about new feeds within the last hour with SFEED_NEW_MAX_SECS=3600 sfeed_plain ~/.sfeed/feeds/* while creating a beautiful web report for last week's news by SFEED_NEW_MAX_SECS=604800 sfeed_html ~/.sfeed/feeds/* --- README | 10 ++++++---- sfeed_curses.1 | 4 ++++ sfeed_curses.c | 11 ++++++++--- sfeed_frames.1 | 7 +++++++ sfeed_frames.c | 15 ++++++++++++--- sfeed_gopher.1 | 4 ++++ sfeed_gopher.c | 15 ++++++++++++--- sfeed_html.1 | 7 +++++++ sfeed_html.c | 15 ++++++++++++--- sfeed_plain.1 | 7 +++++++ sfeed_plain.c | 16 +++++++++++++--- 11 files changed, 92 insertions(+), 19 deletions(-) (limited to 'README') diff --git a/README b/README index 6892e89..313ab4c 100644 --- a/README +++ b/README @@ -289,10 +289,12 @@ Just like the other format programs included in sfeed you can run it like this: sfeed_curses < ~/.sfeed/feeds/xkcd -By default sfeed_curses marks the items of the last day as new/bold. To manage -read/unread items in a different way a plain-text file with a list of the read -URLs can be used. To enable this behaviour the path to this file can be -specified by setting the environment variable $SFEED_URL_FILE to the URL file: +By default sfeed_curses marks the items of the last day as new/bold. This limit +might be overridden by setting the environment variable $SFEED_NEW_MAX_SECS to +the desired maximum in seconds. To manage read/unread items in a different way +a plain-text file with a list of the read URLs can be used. To enable this +behaviour the path to this file can be specified by setting the environment +variable $SFEED_URL_FILE to the URL file: export SFEED_URL_FILE="$HOME/.sfeed/urls" [ -f "$SFEED_URL_FILE" ] || touch "$SFEED_URL_FILE" diff --git a/sfeed_curses.1 b/sfeed_curses.1 index 94ffd4c..b3cccf2 100644 --- a/sfeed_curses.1 +++ b/sfeed_curses.1 @@ -30,6 +30,7 @@ arguments are specified then the data is read from stdin and the feed name is .Pp Items with a timestamp from the last day compared to the system time at the time of loading the feed are marked as new and bold. +This value might be overridden through environment variables. There is also an alternative mode available to mark items as read by matching it against a list of URLs from a plain-text file. Items with an enclosure are marked with a @ symbol. @@ -207,6 +208,9 @@ SIGWINCH. Read and process a sequence of keys as input commands from this environment variable first, afterwards it reads from the tty as usual. This can be useful to automate certain actions at the start. +.It Ev SFEED_NEW_MAX_SECS +Overwrite the maximum age in seconds to mark feeds as new. +By default this is 86400, which equals one day. .It Ev SFEED_PIPER A program where the whole TAB-Separated Value line is piped to. By default this is "sfeed_content". diff --git a/sfeed_curses.c b/sfeed_curses.c index 1abb046..cce608e 100644 --- a/sfeed_curses.c +++ b/sfeed_curses.c @@ -191,6 +191,7 @@ static int plumberia = 0; /* env variable: $SFEED_PLUMBER_INTERACTIVE */ static int piperia = 1; /* env variable: $SFEED_PIPER_INTERACTIVE */ static int yankeria = 0; /* env variable: $SFEED_YANKER_INTERACTIVE */ static int lazyload = 0; /* env variable: $SFEED_LAZYLOAD */ +static int newmaxsecs = 86400; /* env variable: $SFEED_NEW_MAX_SECS */ int ttywritef(const char *fmt, ...) @@ -1322,8 +1323,7 @@ feeds_load(struct feed *feeds, size_t nfeeds) errno = 0; if ((comparetime = time(NULL)) == (time_t)-1) die("time"); - /* 1 day is old news */ - comparetime -= 86400; + comparetime -= newmaxsecs; for (i = 0; i < nfeeds; i++) { f = &feeds[i]; @@ -1967,7 +1967,7 @@ main(int argc, char *argv[]) struct pane *p; struct feed *f; struct row *row; - char *name, *tmp; + char *name, *tmp, *endptr; char *search = NULL; /* search text */ int button, ch, fd, i, keymask, release, x, y; off_t pos; @@ -1997,6 +1997,11 @@ main(int argc, char *argv[]) markunreadcmd = tmp; if ((tmp = getenv("SFEED_LAZYLOAD"))) lazyload = !strcmp(tmp, "1"); + if ((tmp = getenv("SFEED_NEW_MAX_SECS"))) { + newmaxsecs = (int) strtol(tmp, &endptr, 10); + if (*tmp == '\0' || *endptr != '\0' || newmaxsecs <= 0) + err(1, "cannot parse $SFEED_NEW_MAX_SECS"); + } urlfile = getenv("SFEED_URL_FILE"); /* can be NULL */ cmdenv = getenv("SFEED_AUTOCMD"); /* can be NULL */ diff --git a/sfeed_frames.1 b/sfeed_frames.1 index e29498b..976d5ed 100644 --- a/sfeed_frames.1 +++ b/sfeed_frames.1 @@ -23,6 +23,7 @@ file is not written. Items with a timestamp from the last day compared to the system time at the time of formatting are counted and marked as new. Items are marked as new using a bold markup. +This value might be overridden through environment variables. .Pp There is an example style.css stylesheet file included in the distribution. .Sh FILES WRITTEN @@ -37,6 +38,12 @@ feeds. The HTML file of the menu frame which contains navigation "anchor" links (like "#feedname") to the feed names in items.html. .El +.Sh ENVIRONMENT VARIABLES +.Bl -tag -width Ds +.It Ev SFEED_NEW_MAX_SECS +Overwrite the maximum age in seconds to mark feeds as new. +By default this is 86400, which equals one day. +.El .Sh EXIT STATUS .Ex -std .Sh EXAMPLES diff --git a/sfeed_frames.c b/sfeed_frames.c index 178a4a2..e52abd1 100644 --- a/sfeed_frames.c +++ b/sfeed_frames.c @@ -77,8 +77,9 @@ int main(int argc, char *argv[]) { FILE *fpindex, *fpitems, *fpmenu = NULL, *fp; - char *name; + char *name, *tmp, *endptr; int i, showsidebar = (argc > 1); + long l; struct feed *f; if (pledge("stdio rpath wpath cpath", NULL) == -1) @@ -89,8 +90,16 @@ main(int argc, char *argv[]) if ((comparetime = time(NULL)) == (time_t)-1) errx(1, "time"); - /* 1 day is old news */ - comparetime -= 86400; + + if ((tmp = getenv("SFEED_NEW_MAX_SECS"))) { + l = strtol(tmp, &endptr, 10); + if (*tmp == '\0' || *endptr != '\0' || l <= 0) + err(1, "cannot parse $SFEED_NEW_MAX_SECS"); + comparetime -= l; + } else { + /* 1 day is old news */ + comparetime -= 86400; + } /* write main index page */ if (!(fpindex = fopen("index.html", "wb"))) diff --git a/sfeed_gopher.1 b/sfeed_gopher.1 index 43a11c7..f602d2c 100644 --- a/sfeed_gopher.1 +++ b/sfeed_gopher.1 @@ -32,6 +32,7 @@ written to stdout and no files are written. .Pp Items with a timestamp from the last day compared to the system time at the time of formatting are counted and marked as new. +This value might be overridden through environment variables. Items are marked as new with the prefix "N" at the start of the line. .Sh ENVIRONMENT .Bl -tag -width Ds @@ -45,6 +46,9 @@ The default is "127.0.0.1". .It Ev SFEED_GOPHER_PORT This environment variable can be used as the Gopher Port field. The default is "70". +.It Ev SFEED_NEW_MAX_SECS +Overwrite the maximum age in seconds to mark feeds as new. +By default this is 86400, which equals one day. .El .Sh EXIT STATUS .Ex -std diff --git a/sfeed_gopher.c b/sfeed_gopher.c index f62c6ed..007a244 100644 --- a/sfeed_gopher.c +++ b/sfeed_gopher.c @@ -122,8 +122,9 @@ int main(int argc, char *argv[]) { FILE *fpitems, *fpindex, *fp; - char *name, *p; + char *name, *p, *tmp, *endptr; int i; + long l; if (argc == 1) { if (pledge("stdio", NULL) == -1) @@ -139,8 +140,16 @@ main(int argc, char *argv[]) if ((comparetime = time(NULL)) == (time_t)-1) errx(1, "time"); - /* 1 day is old news */ - comparetime -= 86400; + + if ((tmp = getenv("SFEED_NEW_MAX_SECS"))) { + l = strtol(tmp, &endptr, 10); + if (*tmp == '\0' || *endptr != '\0' || l <= 0) + err(1, "cannot parse $SFEED_NEW_MAX_SECS"); + comparetime -= l; + } else { + /* 1 day is old news */ + comparetime -= 86400; + } if ((p = getenv("SFEED_GOPHER_HOST"))) host = p; diff --git a/sfeed_html.1 b/sfeed_html.1 index efeb289..98784c5 100644 --- a/sfeed_html.1 +++ b/sfeed_html.1 @@ -26,9 +26,16 @@ is empty. .Pp Items with a timestamp from the last day compared to the system time at the time of formatting are counted and marked as new. +This value might be overridden through environment variables. Items are marked as new using a bold markup. .Pp There is an example style.css stylesheet file included in the distribution. +.Sh ENVIRONMENT VARIABLES +.Bl -tag -width Ds +.It Ev SFEED_NEW_MAX_SECS +Overwrite the maximum age in seconds to mark feeds as new. +By default this is 86400, which equals one day. +.El .Sh EXIT STATUS .Ex -std .Sh EXAMPLES diff --git a/sfeed_html.c b/sfeed_html.c index ce96687..89506be 100644 --- a/sfeed_html.c +++ b/sfeed_html.c @@ -78,9 +78,10 @@ int main(int argc, char *argv[]) { struct feed *f; - char *name; + char *name, *tmp, *endptr; FILE *fp; int i; + long l; if (pledge(argc == 1 ? "stdio" : "stdio rpath", NULL) == -1) err(1, "pledge"); @@ -89,8 +90,16 @@ main(int argc, char *argv[]) err(1, "calloc"); if ((comparetime = time(NULL)) == (time_t)-1) errx(1, "time"); - /* 1 day is old news */ - comparetime -= 86400; + + if ((tmp = getenv("SFEED_NEW_MAX_SECS"))) { + l = strtol(tmp, &endptr, 10); + if (*tmp == '\0' || *endptr != '\0' || l <= 0) + err(1, "cannot parse $SFEED_NEW_MAX_SECS"); + comparetime -= l; + } else { + /* 1 day is old news */ + comparetime -= 86400; + } fputs("\n" "\n" diff --git a/sfeed_plain.1 b/sfeed_plain.1 index 466ae41..93ed41d 100644 --- a/sfeed_plain.1 +++ b/sfeed_plain.1 @@ -26,6 +26,7 @@ is empty. .Pp Items with a timestamp from the last day compared to the system time at the time of formatting are marked as new. +This value might be overridden through environment variables. Items are marked as new with the prefix "N" at the start of the line. .Pp .Nm @@ -39,6 +40,12 @@ per rune, using .Xr mbtowc 3 and .Xr wcwidth 3 . +.Sh ENVIRONMENT VARIABLES +.Bl -tag -width Ds +.It Ev SFEED_NEW_MAX_SECS +Overwrite the maximum age in seconds to mark feeds as new. +By default this is 86400, which equals one day. +.El .Sh EXIT STATUS .Ex -std .Sh EXAMPLES diff --git a/sfeed_plain.c b/sfeed_plain.c index 8b1f00f..c790ec4 100644 --- a/sfeed_plain.c +++ b/sfeed_plain.c @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -52,8 +53,9 @@ int main(int argc, char *argv[]) { FILE *fp; - char *name; + char *name, *tmp, *endptr; int i; + long l; if (pledge("stdio rpath", NULL) == -1) err(1, "pledge"); @@ -65,8 +67,16 @@ main(int argc, char *argv[]) if ((comparetime = time(NULL)) == (time_t)-1) errx(1, "time"); - /* 1 day is old news */ - comparetime -= 86400; + + if ((tmp = getenv("SFEED_NEW_MAX_SECS"))) { + l = strtol(tmp, &endptr, 10); + if (*tmp == '\0' || *endptr != '\0' || l <= 0) + err(1, "cannot parse $SFEED_NEW_MAX_SECS"); + comparetime -= l; + } else { + /* 1 day is old news */ + comparetime -= 86400; + } if (argc == 1) { printfeed(stdin, ""); -- cgit v1.2.3 From bdcbf8589716c047a732db3cc349ee6114ccc25f Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Sun, 7 May 2023 12:39:26 +0200 Subject: iterate on previous commit which adds $SFEED_NEW_MAX_SECS Separate the common pattern to get the time to compare new items in format tools to the new util function: getcomparetime(). Some changes and notes: - Change it so it is OK to set this value to 0 or negative (in the future). - sfeed_curses: truncating newmaxsecs to an int would limit the value too much. - Just use strtotime() and parse the value to time_t. This is a signed long (32-bit, until 2038) or signed long long (64-bit) on most platforms. - sfeed_curses gets the comparison time on reload aswell and it needs errno = 0, because it uses die(). time() is not guaranteed to set an errno if it fails. - Rename environment variable to $SFEED_NEW_AGE. --- README | 6 +++--- sfeed_curses.1 | 2 +- sfeed_curses.c | 13 +++---------- sfeed_frames.1 | 2 +- sfeed_frames.c | 17 +++-------------- sfeed_gopher.1 | 2 +- sfeed_gopher.c | 17 +++-------------- sfeed_html.1 | 2 +- sfeed_html.c | 17 +++-------------- sfeed_plain.1 | 2 +- sfeed_plain.c | 18 +++--------------- util.c | 18 ++++++++++++++++++ util.h | 1 + 13 files changed, 42 insertions(+), 75 deletions(-) (limited to 'README') diff --git a/README b/README index 313ab4c..427435d 100644 --- a/README +++ b/README @@ -290,9 +290,9 @@ Just like the other format programs included in sfeed you can run it like this: sfeed_curses < ~/.sfeed/feeds/xkcd By default sfeed_curses marks the items of the last day as new/bold. This limit -might be overridden by setting the environment variable $SFEED_NEW_MAX_SECS to -the desired maximum in seconds. To manage read/unread items in a different way -a plain-text file with a list of the read URLs can be used. To enable this +might be overridden by setting the environment variable $SFEED_NEW_AGE to the +desired maximum in seconds. To manage read/unread items in a different way a +plain-text file with a list of the read URLs can be used. To enable this behaviour the path to this file can be specified by setting the environment variable $SFEED_URL_FILE to the URL file: diff --git a/sfeed_curses.1 b/sfeed_curses.1 index b3cccf2..56b2a92 100644 --- a/sfeed_curses.1 +++ b/sfeed_curses.1 @@ -208,7 +208,7 @@ SIGWINCH. Read and process a sequence of keys as input commands from this environment variable first, afterwards it reads from the tty as usual. This can be useful to automate certain actions at the start. -.It Ev SFEED_NEW_MAX_SECS +.It Ev SFEED_NEW_AGE Overwrite the maximum age in seconds to mark feeds as new. By default this is 86400, which equals one day. .It Ev SFEED_PIPER diff --git a/sfeed_curses.c b/sfeed_curses.c index cce608e..b6f995f 100644 --- a/sfeed_curses.c +++ b/sfeed_curses.c @@ -191,7 +191,6 @@ static int plumberia = 0; /* env variable: $SFEED_PLUMBER_INTERACTIVE */ static int piperia = 1; /* env variable: $SFEED_PIPER_INTERACTIVE */ static int yankeria = 0; /* env variable: $SFEED_YANKER_INTERACTIVE */ static int lazyload = 0; /* env variable: $SFEED_LAZYLOAD */ -static int newmaxsecs = 86400; /* env variable: $SFEED_NEW_MAX_SECS */ int ttywritef(const char *fmt, ...) @@ -1321,9 +1320,8 @@ feeds_load(struct feed *feeds, size_t nfeeds) size_t i; errno = 0; - if ((comparetime = time(NULL)) == (time_t)-1) - die("time"); - comparetime -= newmaxsecs; + if ((comparetime = getcomparetime()) == (time_t)-1) + die("getcomparetime"); for (i = 0; i < nfeeds; i++) { f = &feeds[i]; @@ -1967,7 +1965,7 @@ main(int argc, char *argv[]) struct pane *p; struct feed *f; struct row *row; - char *name, *tmp, *endptr; + char *name, *tmp; char *search = NULL; /* search text */ int button, ch, fd, i, keymask, release, x, y; off_t pos; @@ -1997,11 +1995,6 @@ main(int argc, char *argv[]) markunreadcmd = tmp; if ((tmp = getenv("SFEED_LAZYLOAD"))) lazyload = !strcmp(tmp, "1"); - if ((tmp = getenv("SFEED_NEW_MAX_SECS"))) { - newmaxsecs = (int) strtol(tmp, &endptr, 10); - if (*tmp == '\0' || *endptr != '\0' || newmaxsecs <= 0) - err(1, "cannot parse $SFEED_NEW_MAX_SECS"); - } urlfile = getenv("SFEED_URL_FILE"); /* can be NULL */ cmdenv = getenv("SFEED_AUTOCMD"); /* can be NULL */ diff --git a/sfeed_frames.1 b/sfeed_frames.1 index 976d5ed..455c1db 100644 --- a/sfeed_frames.1 +++ b/sfeed_frames.1 @@ -40,7 +40,7 @@ The HTML file of the menu frame which contains navigation "anchor" links (like .El .Sh ENVIRONMENT VARIABLES .Bl -tag -width Ds -.It Ev SFEED_NEW_MAX_SECS +.It Ev SFEED_NEW_AGE Overwrite the maximum age in seconds to mark feeds as new. By default this is 86400, which equals one day. .El diff --git a/sfeed_frames.c b/sfeed_frames.c index e52abd1..b2f75cf 100644 --- a/sfeed_frames.c +++ b/sfeed_frames.c @@ -77,9 +77,8 @@ int main(int argc, char *argv[]) { FILE *fpindex, *fpitems, *fpmenu = NULL, *fp; - char *name, *tmp, *endptr; + char *name; int i, showsidebar = (argc > 1); - long l; struct feed *f; if (pledge("stdio rpath wpath cpath", NULL) == -1) @@ -88,18 +87,8 @@ main(int argc, char *argv[]) if (!(feeds = calloc(argc, sizeof(struct feed)))) err(1, "calloc"); - if ((comparetime = time(NULL)) == (time_t)-1) - errx(1, "time"); - - if ((tmp = getenv("SFEED_NEW_MAX_SECS"))) { - l = strtol(tmp, &endptr, 10); - if (*tmp == '\0' || *endptr != '\0' || l <= 0) - err(1, "cannot parse $SFEED_NEW_MAX_SECS"); - comparetime -= l; - } else { - /* 1 day is old news */ - comparetime -= 86400; - } + if ((comparetime = getcomparetime()) == (time_t)-1) + errx(1, "getcomparetime"); /* write main index page */ if (!(fpindex = fopen("index.html", "wb"))) diff --git a/sfeed_gopher.1 b/sfeed_gopher.1 index f602d2c..bc45121 100644 --- a/sfeed_gopher.1 +++ b/sfeed_gopher.1 @@ -46,7 +46,7 @@ The default is "127.0.0.1". .It Ev SFEED_GOPHER_PORT This environment variable can be used as the Gopher Port field. The default is "70". -.It Ev SFEED_NEW_MAX_SECS +.It Ev SFEED_NEW_AGE Overwrite the maximum age in seconds to mark feeds as new. By default this is 86400, which equals one day. .El diff --git a/sfeed_gopher.c b/sfeed_gopher.c index 007a244..78e1f12 100644 --- a/sfeed_gopher.c +++ b/sfeed_gopher.c @@ -122,9 +122,8 @@ int main(int argc, char *argv[]) { FILE *fpitems, *fpindex, *fp; - char *name, *p, *tmp, *endptr; + char *name, *p; int i; - long l; if (argc == 1) { if (pledge("stdio", NULL) == -1) @@ -138,18 +137,8 @@ main(int argc, char *argv[]) err(1, "pledge"); } - if ((comparetime = time(NULL)) == (time_t)-1) - errx(1, "time"); - - if ((tmp = getenv("SFEED_NEW_MAX_SECS"))) { - l = strtol(tmp, &endptr, 10); - if (*tmp == '\0' || *endptr != '\0' || l <= 0) - err(1, "cannot parse $SFEED_NEW_MAX_SECS"); - comparetime -= l; - } else { - /* 1 day is old news */ - comparetime -= 86400; - } + if ((comparetime = getcomparetime()) == (time_t)-1) + errx(1, "getcomparetime"); if ((p = getenv("SFEED_GOPHER_HOST"))) host = p; diff --git a/sfeed_html.1 b/sfeed_html.1 index 98784c5..e517a63 100644 --- a/sfeed_html.1 +++ b/sfeed_html.1 @@ -32,7 +32,7 @@ Items are marked as new using a bold markup. There is an example style.css stylesheet file included in the distribution. .Sh ENVIRONMENT VARIABLES .Bl -tag -width Ds -.It Ev SFEED_NEW_MAX_SECS +.It Ev SFEED_NEW_AGE Overwrite the maximum age in seconds to mark feeds as new. By default this is 86400, which equals one day. .El diff --git a/sfeed_html.c b/sfeed_html.c index 89506be..9269413 100644 --- a/sfeed_html.c +++ b/sfeed_html.c @@ -78,28 +78,17 @@ int main(int argc, char *argv[]) { struct feed *f; - char *name, *tmp, *endptr; + char *name; FILE *fp; int i; - long l; if (pledge(argc == 1 ? "stdio" : "stdio rpath", NULL) == -1) err(1, "pledge"); if (!(feeds = calloc(argc, sizeof(struct feed)))) err(1, "calloc"); - if ((comparetime = time(NULL)) == (time_t)-1) - errx(1, "time"); - - if ((tmp = getenv("SFEED_NEW_MAX_SECS"))) { - l = strtol(tmp, &endptr, 10); - if (*tmp == '\0' || *endptr != '\0' || l <= 0) - err(1, "cannot parse $SFEED_NEW_MAX_SECS"); - comparetime -= l; - } else { - /* 1 day is old news */ - comparetime -= 86400; - } + if ((comparetime = getcomparetime()) == (time_t)-1) + errx(1, "getcomparetime"); fputs("\n" "\n" diff --git a/sfeed_plain.1 b/sfeed_plain.1 index 93ed41d..2e8e6ad 100644 --- a/sfeed_plain.1 +++ b/sfeed_plain.1 @@ -42,7 +42,7 @@ and .Xr wcwidth 3 . .Sh ENVIRONMENT VARIABLES .Bl -tag -width Ds -.It Ev SFEED_NEW_MAX_SECS +.It Ev SFEED_NEW_AGE Overwrite the maximum age in seconds to mark feeds as new. By default this is 86400, which equals one day. .El diff --git a/sfeed_plain.c b/sfeed_plain.c index c790ec4..f8ce7ec 100644 --- a/sfeed_plain.c +++ b/sfeed_plain.c @@ -2,7 +2,6 @@ #include #include -#include #include #include @@ -53,9 +52,8 @@ int main(int argc, char *argv[]) { FILE *fp; - char *name, *tmp, *endptr; + char *name; int i; - long l; if (pledge("stdio rpath", NULL) == -1) err(1, "pledge"); @@ -65,18 +63,8 @@ main(int argc, char *argv[]) if (pledge(argc == 1 ? "stdio" : "stdio rpath", NULL) == -1) err(1, "pledge"); - if ((comparetime = time(NULL)) == (time_t)-1) - errx(1, "time"); - - if ((tmp = getenv("SFEED_NEW_MAX_SECS"))) { - l = strtol(tmp, &endptr, 10); - if (*tmp == '\0' || *endptr != '\0' || l <= 0) - err(1, "cannot parse $SFEED_NEW_MAX_SECS"); - comparetime -= l; - } else { - /* 1 day is old news */ - comparetime -= 86400; - } + if ((comparetime = getcomparetime()) == (time_t)-1) + errx(1, "getcomparetime"); if (argc == 1) { printfeed(stdin, ""); diff --git a/util.c b/util.c index 0b7da06..e5aec3b 100644 --- a/util.c +++ b/util.c @@ -318,6 +318,24 @@ strtotime(const char *s, time_t *t) return 0; } +time_t +getcomparetime(void) +{ + time_t now, t; + char *p; + + if ((now = time(NULL)) == (time_t)-1) + return (time_t)-1; + + if ((p = getenv("SFEED_NEW_AGE"))) { + if (strtotime(p, &t) == -1) + return (time_t)-1; + return now - t; + } + + return now - 86400; /* 1 day is old news */ +} + /* Escape characters below as HTML 2.0 / XML 1.0. */ void xmlencode(const char *s, FILE *fp) diff --git a/util.h b/util.h index 9138de7..3ce5e41 100644 --- a/util.h +++ b/util.h @@ -71,6 +71,7 @@ int uri_makeabs(struct uri *, struct uri *, struct uri *); int uri_parse(const char *, struct uri *); void checkfileerror(FILE *, const char *, int); +time_t getcomparetime(void); void parseline(char *, char *[FieldLast]); void printutf8pad(FILE *, const char *, size_t, int); int strtotime(const char *, time_t *); -- cgit v1.2.3 From 6245c2866bd38dcad51833f39677f99fb25fd0ef Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Tue, 16 May 2023 18:30:09 +0200 Subject: README: RSS 0.90+ is supported, not 0.91+ See also: https://www.rssboard.org/rss-0-9-0 --- README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'README') diff --git a/README b/README index 427435d..fbe40b6 100644 --- a/README +++ b/README @@ -141,7 +141,7 @@ sfeed supports a subset of XML 1.0 and a subset of: - Atom 1.0 (RFC 4287): https://datatracker.ietf.org/doc/html/rfc4287 - Atom 0.3 (draft, historic). -- RSS 0.91+. +- RSS 0.90+. - RDF (when used with RSS). - MediaRSS extensions (media:). - Dublin Core extensions (dc:). -- cgit v1.2.3 From 96d7afc7d7511f05ba07c5acbb5bbfb2847bc126 Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Fri, 9 Jun 2023 14:40:40 +0200 Subject: sfeed_update/sfeedrc: add url as parameter to the filter() and order() function This might make it easier to set filters or ordering by pattern matching on a group of feeds by the feed URL. For example youtube or reddit feeds. Another way which was already possible is prefixing names with for example: "reddit somename" or "yt somename". --- README | 2 +- sfeed_update | 8 ++++---- sfeedrc.5 | 10 +++++++--- 3 files changed, 12 insertions(+), 8 deletions(-) (limited to 'README') diff --git a/README b/README index fbe40b6..d128360 100644 --- a/README +++ b/README @@ -336,7 +336,7 @@ filtering items per feed. It can be used to shorten URLs, filter away advertisements, strip tracking parameters and more. # filter fields. - # filter(name) + # filter(name, url) filter() { case "$1" in "tweakers") diff --git a/sfeed_update b/sfeed_update index 3da45d3..dadea9b 100755 --- a/sfeed_update +++ b/sfeed_update @@ -71,7 +71,7 @@ parse() { } # filter fields. -# filter(name) +# filter(name, url) filter() { cat } @@ -83,7 +83,7 @@ merge() { } # order by timestamp (descending). -# order(name) +# order(name, url) order() { sort -t ' ' -k1rn,1 } @@ -124,7 +124,7 @@ _feed() { fi rm -f "${tmpfeedfile}.utf8" - if ! filter "${name}" < "${tmpfeedfile}.tsv" > "${tmpfeedfile}.filter"; then + if ! filter "${name}" "${feedurl}" < "${tmpfeedfile}.tsv" > "${tmpfeedfile}.filter"; then log_error "${name}" "FAIL (FILTER)" return 1 fi @@ -142,7 +142,7 @@ _feed() { fi rm -f "${tmpfeedfile}.filter" - if ! order "${name}" < "${tmpfeedfile}.merge" > "${tmpfeedfile}.order"; then + if ! order "${name}" "${feedurl}" < "${tmpfeedfile}.merge" > "${tmpfeedfile}.order"; then log_error "${name}" "FAIL (ORDER)" return 1 fi diff --git a/sfeedrc.5 b/sfeedrc.5 index 91f82ce..16e0641 100644 --- a/sfeedrc.5 +++ b/sfeedrc.5 @@ -1,4 +1,4 @@ -.Dd January 18, 2023 +.Dd June 9, 2023 .Dt SFEEDRC 5 .Os .Sh NAME @@ -101,13 +101,15 @@ URL of the feed. Base URL of the feed links. This argument allows to fix relative item links. .El -.It Fn filter "name" +.It Fn filter "name" "url" Filter .Xr sfeed 5 data from stdin and write it to stdout, its arguments are: .Bl -tag -width Ds .It Fa name Feed name. +.It Fa url +URL of the feed. .El .It Fn merge "name" "oldfile" "newfile" Merge @@ -121,13 +123,15 @@ Old file. .It Fa newfile New file. .El -.It Fn order "name" +.It Fn order "name" "url" Sort .Xr sfeed 5 data from stdin and write it to stdout, its arguments are: .Bl -tag -width Ds .It Fa name Feed name. +.It Fa url +URL of the feed. .El .El .Sh EXAMPLES -- cgit v1.2.3 From e96bdb644ae2f49125079e6218e2777caf633e1c Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Fri, 9 Jun 2023 14:44:34 +0200 Subject: README: improve note about CDNs and user-agent blocking Prompted by a question from a user. Specificly mention reddit.com now blocks clients which don't set the User-Agent header. It is specificly mentioned because there is a reddit feed in the example sfeedrc. --- README | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'README') diff --git a/README b/README index d128360..8098a76 100644 --- a/README +++ b/README @@ -679,8 +679,8 @@ additional metadata from the previous request. CDNs blocking requests due to a missing HTTP User-Agent request header sfeed_update will not send the "User-Agent" header by default for privacy -reasons. Some CDNs like Cloudflare don't like this and will block such HTTP -requests. +reasons. Some CDNs like Cloudflare or websites like Reddit.com don't like this +and will block such HTTP requests. A custom User-Agent can be set by using the curl -H option, like so: -- cgit v1.2.3 From 63308527f5197ddbcad6b06c5c1bbaf12f997e57 Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Sat, 9 Dec 2023 21:51:45 +0100 Subject: improve compatibility with zsh as a non-interactive shell In zsh the variables $path and $status are special. https://zsh.sourceforge.io/Doc/Release/Parameters.html#index-path https://zsh.sourceforge.io/Doc/Release/Parameters.html#index-status (No promises I will keep up with this insanity in the future though) --- README | 6 +++--- sfeed_opml_export | 8 ++++---- sfeed_update | 12 ++++++------ 3 files changed, 13 insertions(+), 13 deletions(-) (limited to 'README') diff --git a/README b/README index 8098a76..85a3fb9 100644 --- a/README +++ b/README @@ -752,12 +752,12 @@ sfeed_update_xargs shellscript: mkdir -p "${sfeedpath}" # print feeds for parallel processing with xargs. feeds | SFEED_UPDATE_CHILD="1" xargs -r -0 -P "${maxjobs}" -L 6 "$(readlink -f "$0")" - status=$? + statuscode=$? # check error exit status indicator for parallel jobs. - test -f "${sfeedtmpdir}/ok" || status=1 + test -f "${sfeedtmpdir}/ok" || statuscode=1 # cleanup temporary files etc. cleanup - exit ${status} + exit ${statuscode} - - - diff --git a/sfeed_opml_export b/sfeed_opml_export index 7c96d5d..6420d5e 100755 --- a/sfeed_opml_export +++ b/sfeed_opml_export @@ -7,16 +7,16 @@ loadconfig() { if [ "$1" != "" ]; then # get absolute path of config file required for including. config="$1" - path=$(readlink -f "${config}" 2>/dev/null) + configpath=$(readlink -f "${config}" 2>/dev/null) else # default config location. config="$HOME/.sfeed/sfeedrc" - path="${config}" + configpath="${config}" fi # config is loaded here to be able to override $sfeedpath or functions. - if [ -r "${path}" ]; then - . "${path}" + if [ -r "${configpath}" ]; then + . "${configpath}" else printf "Configuration file \"%s\" cannot be read.\n" "${config}" >&2 echo "See the sfeedrc.example file or the sfeedrc(5) man page for an example." >&2 diff --git a/sfeed_update b/sfeed_update index dadea9b..2b7d89b 100755 --- a/sfeed_update +++ b/sfeed_update @@ -16,16 +16,16 @@ loadconfig() { if [ "$1" != "" ]; then # get absolute path of config file required for including. config="$1" - path=$(readlink -f "${config}" 2>/dev/null) + configpath=$(readlink -f "${config}" 2>/dev/null) else # default config location. config="$HOME/.sfeed/sfeedrc" - path="${config}" + configpath="${config}" fi # config is loaded here to be able to override $sfeedpath or functions. - if [ -r "${path}" ]; then - . "${path}" + if [ -r "${configpath}" ]; then + . "${configpath}" else printf "Configuration file \"%s\" cannot be read.\n" "${config}" >&2 echo "See the sfeedrc.example file or the sfeedrc(5) man page for an example." >&2 @@ -214,12 +214,12 @@ main() { [ ${signo} -eq 0 ] && wait # check error exit status indicator for parallel jobs. [ -f "${sfeedtmpdir}/ok" ] - status=$? + statuscode=$? # cleanup temporary files etc. cleanup # on signal SIGINT and SIGTERM exit with signal number + 128. [ ${signo} -ne 0 ] && exit $((signo+128)) - exit ${status} + exit ${statuscode} } [ "${SFEED_UPDATE_INCLUDE}" = "1" ] || main "$@" -- cgit v1.2.3 From 7526bd914c314dd86cc7386bbfaf55ad0d130130 Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Fri, 15 Dec 2023 13:48:49 +0100 Subject: README: sfeed_opml_export uses awk --- README | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'README') diff --git a/README b/README index 85a3fb9..8ea13ed 100644 --- a/README +++ b/README @@ -108,7 +108,8 @@ Optional dependencies - POSIX sh(1), used by sfeed_update(1) and sfeed_opml_export(1). - POSIX utilities such as awk(1) and sort(1), - used by sfeed_content(1), sfeed_markread(1) and sfeed_update(1). + used by sfeed_content(1), sfeed_markread(1), sfeed_opml_export(1) and + sfeed_update(1). - curl(1) binary: https://curl.haxx.se/ , used by sfeed_update(1), but can be replaced with any tool like wget(1), OpenBSD ftp(1) or hurl(1): https://git.codemadness.org/hurl/ -- cgit v1.2.3 From 89dbf565de7b980d2b336c58633b7a27681adaf5 Mon Sep 17 00:00:00 2001 From: Hiltjo Posthuma Date: Tue, 26 Dec 2023 16:37:21 +0100 Subject: README: update dependency information for xargs for sfeed_update Remove the parallel xargs example because it is now the default. --- README | 58 ++-------------------------------------------------------- 1 file changed, 2 insertions(+), 56 deletions(-) (limited to 'README') diff --git a/README b/README index 8ea13ed..47d4d76 100644 --- a/README +++ b/README @@ -117,6 +117,8 @@ Optional dependencies used by sfeed_update(1). If the text in your RSS/Atom feeds are already UTF-8 encoded then you don't need this. For a minimal iconv implementation: https://git.etalabs.net/cgit/noxcuse/tree/src/iconv.c +- xargs with support for the -P and -0 option, + used by sfeed_update(1). - mandoc for documentation: https://mdocml.bsd.lv/ - curses (typically ncurses), otherwise see minicurses.h, used by sfeed_curses(1). @@ -706,62 +708,6 @@ sfeedrc file and change the curl options "-L --max-redirs 0". - - - -Shellscript to update feeds in parallel more efficiently using xargs -P. - -It creates a queue of the feeds with its settings, then uses xargs to process -them in parallel using the common, but non-POSIX -P option. This is more -efficient than the more portable solution in sfeed_update which can stall a -batch of $maxjobs in the queue if one item is slow. - -sfeed_update_xargs shellscript: - - #!/bin/sh - # update feeds, merge with old feeds using xargs in parallel mode (non-POSIX). - - # include script and reuse its functions, but do not start main(). - SFEED_UPDATE_INCLUDE="1" . sfeed_update - # load config file, sets $config. - loadconfig "$1" - - # process a single feed. - # args are: config, tmpdir, name, feedurl, basesiteurl, encoding - if [ "${SFEED_UPDATE_CHILD}" = "1" ]; then - sfeedtmpdir="$2" - _feed "$3" "$4" "$5" "$6" - exit $? - fi - - # ...else parent mode: - - # feed(name, feedurl, basesiteurl, encoding) - feed() { - # workaround: *BSD xargs doesn't handle empty fields in the middle. - name="${1:-$$}" - feedurl="${2:-http://}" - basesiteurl="${3:-${feedurl}}" - encoding="$4" - - printf '%s\0%s\0%s\0%s\0%s\0%s\0' "${config}" "${sfeedtmpdir}" \ - "${name}" "${feedurl}" "${basesiteurl}" "${encoding}" - } - - # fetch feeds and store in temporary directory. - sfeedtmpdir="$(mktemp -d '/tmp/sfeed_XXXXXX')" || exit 1 - mkdir -p "${sfeedtmpdir}/feeds" - touch "${sfeedtmpdir}/ok" || exit 1 - # make sure path exists. - mkdir -p "${sfeedpath}" - # print feeds for parallel processing with xargs. - feeds | SFEED_UPDATE_CHILD="1" xargs -r -0 -P "${maxjobs}" -L 6 "$(readlink -f "$0")" - statuscode=$? - # check error exit status indicator for parallel jobs. - test -f "${sfeedtmpdir}/ok" || statuscode=1 - # cleanup temporary files etc. - cleanup - exit ${statuscode} - -- - - - Shellscript to handle URLs and enclosures in parallel using xargs -P. This can be used to download and process URLs for downloading podcasts, -- cgit v1.2.3