commit 2f8a83288d91ea0abc2e4ebd6754513ee3ad37ec
parent 4d9f922c8396bada73fb0b1e318c8b947f0f606b
Author: Hiltjo Posthuma <hiltjo@codemadness.org>
Date:   Mon, 25 Jan 2021 19:36:41 +0100

README: add an example script to reuse the sfeed_update code

This code uses the non-portable xargs -P option to more efficiently process
feeds in parallel.

Diffstat:
MREADME | 43+++++++++++++++++++++++++++++++++++++++++++
1 file changed, 43 insertions(+), 0 deletions(-)

diff --git a/README b/README @@ -626,6 +626,49 @@ sfeedrc file and change the curl options "-L --max-redirs 0". - - - +Shellscript to update feeds in parallel more efficiently using xargs -P. + +It creates a queue of the feeds with its settings, then uses xargs to process +them in parallel using the common, but non-POSIX -P option. This is more +efficient than the more portable solution in sfeed_update which can stall a +batch of $maxjobs in the queue if one item is slow. + +sfeed_update_xargs shellscript: + + #!/bin/sh + # update feeds, merge with old feeds using xargs in parallel mode (non-POSIX). + + # include script and reuse its functions, but do not start main(). + SFEED_UPDATE_INCLUDE="1" . sfeed_update + # load config file, sets $config. + loadconfig "$1" + + # process a single feed. + # args are: config, tmpdir, name, feedurl, basesiteurl, encoding + if [ "${SFEED_UPDATE_CHILD}" != "" ]; then + sfeedtmpdir="$2" + _feed "$3" "$4" "$5" "$6" + exit $? + fi + + # ...else parent mode: + + # feed(name, feedurl, basesiteurl, encoding) + feed() { + printf '%s\0%s\0%s\0%s\0%s\0%s\0' "${config}" "${sfeedtmpdir}" "$1" "$2" "$3" "$4" + } + + # fetch feeds and store in temporary directory. + sfeedtmpdir="$(mktemp -d '/tmp/sfeed_XXXXXX')" + # make sure path exists. + mkdir -p "${sfeedpath}" + # print feeds for parallel processing with xargs. + feeds | SFEED_UPDATE_CHILD="1" xargs -0 -P "${maxjobs}" -L 6 "$(readlink -f "$0")" + # cleanup temporary files etc. + cleanup + +- - - + Shellscript to export existing newsboat cached items from sqlite3 to the sfeed TSV format.