From 522ac62f54cedad9713ef2166bd11924df2f17f8 Mon Sep 17 00:00:00 2001 From: gravel Date: Sat, 27 May 2023 13:21:48 +0000 Subject: [PATCH 1/8] Monorepo decomposition --- .phpenv | 19 +--- Makefile | 10 +- listings/listings.ini | 106 ++++++++++++++++++++ php/fetch-servers.php | 1 + php/generate-html.php | 1 + php/listing-providers/generate-listings.php | 5 + php/utils/getopt.php | 17 ++++ 7 files changed, 138 insertions(+), 21 deletions(-) create mode 100644 listings/listings.ini create mode 100644 php/listing-providers/generate-listings.php create mode 100644 php/utils/getopt.php diff --git a/.phpenv b/.phpenv index 968940f..0ee1220 100644 --- a/.phpenv +++ b/.phpenv @@ -11,23 +11,10 @@ $ROOM_ICONS="$DOCUMENT_ROOT/icons"; $ROOM_ICONS_RELATIVE="icons"; - $REPOSITORY_CANONICAL_URL="https://codeberg.org/gravel/sessioncommunities.online"; - - include_once "$PROJECT_ROOT/php/utils/logging.php"; - - // Read the -v|--verbose option increasing logging verbosity to debug. - $options = getopt("v", ["verbose", "fast", "no-color"]); - if (isset($options["v"]) or isset($options["verbose"])) { - $LOGGING_VERBOSITY = LoggingVerbosity::Debug; - } + $LISTING_PROVIDER_ROOT="$PROJECT_ROOT/listings"; + $LISTING_PROVIDER_OUTPUT="$LISTING_PROVIDER_ROOT/lp-output"; - if (isset($options["fast"])) { - $FAST_FETCH_MODE = true; - } - - if (isset($options["no-color"])) { - LoggingVerbosity::$showColor = false; - } + $REPOSITORY_CANONICAL_URL="https://codeberg.org/gravel/sessioncommunities.online"; // set timeout for file_get_contents() ini_set('default_socket_timeout', 6); // in seconds, default is 60 diff --git a/Makefile b/Makefile index 7926096..627653e 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ list: ## Using make dependencies is duplicating behaviour but reads better. # /bin/php php/update-listing.php # Refresh listing and generate HTML. -all: fetch html +sco: fetch html # Fetch room listing. fetch: @@ -60,19 +60,19 @@ clean: # Build everything from scratch and test functionality. test: FLAGS = --verbose -test: clean all open server +test: clean sco open server # Build everything from scratch and test functionality on LAN. test-lan: FLAGS = --verbose -test-lan: clean all open lan-server +test-lan: clean sco open lan-server # Run basic tests without launching site in browser. test-noninteractive: FLAGS = --verbose -test-noninteractive: clean all +test-noninteractive: clean sco # Run Continuous Integration tests. test-ci: FLAGS = --verbose --fast --no-color -test-ci: clean all +test-ci: clean sco # -- Aliases -- serve: server diff --git a/listings/listings.ini b/listings/listings.ini new file mode 100644 index 0000000..844eaa5 --- /dev/null +++ b/listings/listings.ini @@ -0,0 +1,106 @@ + +[regional] +name=Regional Communities +rating=safe +rooms[]=brasil+118d +rooms[]=deutsch+118d +rooms[]=espanol+118d +rooms[]=francais+118d +rooms[]=iom+118d +rooms[]=italiano+118d +rooms[]=nl+118d +rooms[]=persian+118d +rooms[]=portugues+118d +rooms[]=uk+118d +rooms[]=russian+118d +rooms[]=bangladesh+13f6 +rooms[]=czsk+13f6 +rooms[]=india+13f6 +rooms[]=indonesia+13f6 +rooms[]=korea+13f6 +rooms[]=poland+13f6 +rooms[]=scandinavia+2812 +rooms[]=fi+2812 +rooms[]=br+2b92 +rooms[]=germanpoliticssocietyphilosophy+8183 +sogs[]=fe93941471c07f294255391dba92ae3cf356efc4fdd287d8ba1ffef096dbaf56 + +[tech] +name=Tech Communities +rating=safe +rooms[]=android+118d +rooms[]=ai+118d +rooms[]=gaming+118d +rooms[]=im+118d +rooms[]=linux+118d +rooms[]=webdev+118d +rooms[]=programming+13f6 +rooms[]=tech+2054 +rooms[]=ev-en+2812 +rooms[]=pv-en+2812 +rooms[]=gee+2b92 +rooms[]=mathnodes-dvpn-oxen-dero+8585 +rooms[]=plex+c01b + +[china] +na;=Chinese-language Communities +rating=unknown +sogs[]=45674f4135e2dd929279a72e1e73957eed741c7bec00c87b1797ad52c235ea36 +sogs[]=7f55510a392324988041e7a97191f618a32dc05f2f6343de917fcfe1cb424e51 + +[powerusers] +name=Communities for Powerusers +rating=safe +rooms[]=snops+118d +rooms[]=sogops+118d +rooms[]=indexloki+2cbd +rooms[]=SNappsDevs+2cbd +rooms[]=sessiondev+e56f + +[misc] +name=Miscellanous +rating=unknown +rooms[]=health+118d +rooms[]=music+118d +rooms[]=modernsurvival+118d +rooms[]=philosophy+118d +rooms[]=pol+2054 +rooms[]=biz+2b92 +rooms[]=guns+2b92 +rooms[]=hall+7242 +rooms[]=camping+c01b +rooms[]=offtopic+e56f +rooms[]=AISFW+fc30 + +[oxen] +name=Official Communities +rating=safe +sogs[]=a03c + +[cryptocurrency] +name=Cryptocurrency +rooms[]=bitcoinaustralia+4bec +rooms[]=moneroaustralia+4bec +rooms[]=crypto+a03c + +[privsec] +name=Privacy & Security +rating=safe +rooms[]=im+118d +rooms[]=privacy+118d +rooms[]=security+118d +rooms[]=degoogle+48e9 +rooms[]=privacyaustralia+4bec + +[casual] +name=Casual Communities +rating=safe +rooms[]=general-chat+8c12 +rooms[]=midnight-madness+9be4 + +[nsfw] +name=18+ Communities +rating=nsfw +rooms[]=womanbodybeauty+13f6 +rooms[]=piao+2cd5 +rooms[]=AINSFW+fc30 \ No newline at end of file diff --git a/php/fetch-servers.php b/php/fetch-servers.php index 0bdcfb8..e0e01b3 100644 --- a/php/fetch-servers.php +++ b/php/fetch-servers.php @@ -2,6 +2,7 @@ // requires php-curl require_once 'getenv.php'; + require_once 'utils/getopt.php'; require_once 'utils/utils.php'; require_once 'servers/known-servers.php'; require_once 'utils/servers-rooms.php'; diff --git a/php/generate-html.php b/php/generate-html.php index 9034ccc..6cd874c 100644 --- a/php/generate-html.php +++ b/php/generate-html.php @@ -2,6 +2,7 @@ // Perform static site generation. require_once "getenv.php"; + require_once "utils/getopt.php"; // https://stackoverflow.com/a/17161106 function rglob($pattern, $flags = 0) { diff --git a/php/listing-providers/generate-listings.php b/php/listing-providers/generate-listings.php new file mode 100644 index 0000000..6fef4ab --- /dev/null +++ b/php/listing-providers/generate-listings.php @@ -0,0 +1,5 @@ + \ No newline at end of file diff --git a/php/utils/getopt.php b/php/utils/getopt.php new file mode 100644 index 0000000..654e7c1 --- /dev/null +++ b/php/utils/getopt.php @@ -0,0 +1,17 @@ + \ No newline at end of file From 89c3c283d881f866bd00e99658dd586e58b194b8 Mon Sep 17 00:00:00 2001 From: gravel Date: Thu, 1 Jun 2023 15:16:01 +0000 Subject: [PATCH 2/8] Fix typo in listing config --- listings/listings.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/listings/listings.ini b/listings/listings.ini index 844eaa5..14bfbee 100644 --- a/listings/listings.ini +++ b/listings/listings.ini @@ -43,7 +43,7 @@ rooms[]=mathnodes-dvpn-oxen-dero+8585 rooms[]=plex+c01b [china] -na;=Chinese-language Communities +name=Chinese-language Communities rating=unknown sogs[]=45674f4135e2dd929279a72e1e73957eed741c7bec00c87b1797ad52c235ea36 sogs[]=7f55510a392324988041e7a97191f618a32dc05f2f6343de917fcfe1cb424e51 From 178e091aa11ac440fd563a5082ca0268ff29e17b Mon Sep 17 00:00:00 2001 From: gravel Date: Thu, 1 Jun 2023 16:00:21 +0000 Subject: [PATCH 3/8] Implement basic listings --- .gitignore | 3 +++ .phpenv | 3 +++ Makefile | 4 ++++ php/listing-providers/generate-listings.php | 5 ----- php/utils/servers-rooms.php | 13 +++++++++++++ 5 files changed, 23 insertions(+), 5 deletions(-) delete mode 100644 php/listing-providers/generate-listings.php diff --git a/.gitignore b/.gitignore index 7bb4750..75327c4 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,9 @@ output/qr-codes # Served icons output/icons +# Served listings +listings/lp-output + # Server-side cache cache diff --git a/.phpenv b/.phpenv index 0ee1220..ea74889 100644 --- a/.phpenv +++ b/.phpenv @@ -12,7 +12,10 @@ $ROOM_ICONS_RELATIVE="icons"; $LISTING_PROVIDER_ROOT="$PROJECT_ROOT/listings"; + $LISTINGS_INI="$LISTING_PROVIDER_ROOT/listings.ini"; $LISTING_PROVIDER_OUTPUT="$LISTING_PROVIDER_ROOT/lp-output"; + $LISTING_PROVIDER_LISTING_SUMMARY="$LISTING_PROVIDER_OUTPUT/listings"; + $LISTING_PROVIDER_LISTINGS="$LISTING_PROVIDER_OUTPUT/listing"; $REPOSITORY_CANONICAL_URL="https://codeberg.org/gravel/sessioncommunities.online"; diff --git a/Makefile b/Makefile index 627653e..87a57dd 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,10 @@ fetch: html: /bin/php php/generate-html.php $(FLAGS) +# Generate listing provider endpoints from data. +listing: + /bin/php php/generate-listings.php $(FLAGS) + # Serve a local copy which responds to file changes. dev: FLAGS = --verbose dev: open diff --git a/php/listing-providers/generate-listings.php b/php/listing-providers/generate-listings.php deleted file mode 100644 index 6fef4ab..0000000 --- a/php/listing-providers/generate-listings.php +++ /dev/null @@ -1,5 +0,0 @@ - \ No newline at end of file diff --git a/php/utils/servers-rooms.php b/php/utils/servers-rooms.php index cf96858..7edf421 100644 --- a/php/utils/servers-rooms.php +++ b/php/utils/servers-rooms.php @@ -113,6 +113,19 @@ return $details; } + /** + * Return information for JSON serialization in listing. + */ + function to_listing_data(): array { + $details = get_object_vars($this); + $details['join_url'] = $this->get_join_url(); + $details['tags'] = array_map(function(\CommunityTag $tag) { + return $tag->text; + }, $this->tags); + unset($details['message_sequence']); + return $details; + } + /** * Create a CommunityRoom instance from loaded data. * @param CommunityServer $server From 45fc0deb13143b7ba76fc8d624c790808db41575 Mon Sep 17 00:00:00 2001 From: gravel Date: Thu, 1 Jun 2023 16:00:48 +0000 Subject: [PATCH 4/8] Fix official SOGS public key --- listings/listings.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/listings/listings.ini b/listings/listings.ini index 14bfbee..1a0c8d6 100644 --- a/listings/listings.ini +++ b/listings/listings.ini @@ -75,7 +75,7 @@ rooms[]=AISFW+fc30 [oxen] name=Official Communities rating=safe -sogs[]=a03c +sogs[]=a03c383cf63c3c4efe67acc52112a6dd734b3a946b9545f488aaa93da7991238 [cryptocurrency] name=Cryptocurrency From 2e60c756952d0f1efd95739f91e5412f468c77df Mon Sep 17 00:00:00 2001 From: gravel Date: Thu, 1 Jun 2023 16:10:52 +0000 Subject: [PATCH 5/8] Remove redundant listing data --- php/utils/servers-rooms.php | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/php/utils/servers-rooms.php b/php/utils/servers-rooms.php index 7edf421..91b1ae9 100644 --- a/php/utils/servers-rooms.php +++ b/php/utils/servers-rooms.php @@ -118,12 +118,18 @@ */ function to_listing_data(): array { $details = get_object_vars($this); + unset($details['server']); $details['join_url'] = $this->get_join_url(); - $details['tags'] = array_map(function(\CommunityTag $tag) { - return $tag->text; - }, $this->tags); - unset($details['message_sequence']); - return $details; + unset($details['tags']); + unset($details['language_flag']); + return array( + "api" => $details, + "extras" => array( + "join_url" => $this->get_join_url(), + "language_flag" => $this->language_flag, + "tags" => $this->get_raw_tags() + ) + ); } /** @@ -317,6 +323,15 @@ return count($this->get_staff()) < $minimal_staff_count; } + /** + * @return string[] + */ + function get_raw_tags(): array { + return array_map(function(\CommunityTag $tag) { + return $tag->text; + }, $this->tags); + } + /** * Return the tags associated with this room. * @return \CommunityTag[] Tags as string array. From 2953d022e4f2b10a7257da8ab30d010058b6b8e2 Mon Sep 17 00:00:00 2001 From: gravel Date: Thu, 1 Jun 2023 16:11:22 +0000 Subject: [PATCH 6/8] Add untracked file --- php/generate-listings.php | 116 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 php/generate-listings.php diff --git a/php/generate-listings.php b/php/generate-listings.php new file mode 100644 index 0000000..1e962d6 --- /dev/null +++ b/php/generate-listings.php @@ -0,0 +1,116 @@ +name = $name; + $this->rating = $rating ?? "unknown"; + $this->rooms = $rooms; + } + + public function jsonSerialize(): mixed { + // TODO: Careful serialization + $details = get_object_vars($this); + $details['rooms'] = array_map(function(\CommunityRoom $room){ + return $room->to_listing_data(); + }, $this->rooms); + return $details; + } + + public function to_summary(): array { + return array( + 'name' => $this->name, + 'rating' => $this->rating, + 'rooms' => count($this->rooms) + ); + } + } + + /** + * @return \CommunityListing[] + */ + function resolve_listings_config(): array { + global $LISTINGS_INI, $ROOMS_FILE; + $listings_raw = parse_ini_file($LISTINGS_INI, process_sections: true); + $servers_raw = file_get_contents($ROOMS_FILE); + $server_data = json_decode($servers_raw, true); + $servers = CommunityServer::from_details_array($server_data); + $rooms_by_id = []; + foreach (CommunityServer::enumerate_rooms($servers) as $room) { + $rooms_by_id[$room->get_room_identifier()] = $room; + } + $sogs_by_pubkey = []; + foreach ($servers as $server) { + $sogs_by_pubkey[$server->get_pubkey()] = $server; + } + $listings = []; + foreach ($listings_raw as $id => $listing_props) { + $rooms = []; + if (isset($listing_props['rooms'])) { + foreach ($listing_props['rooms'] as $room_id) { + if (isset($rooms_by_id[$room_id])) { + $rooms[] = $rooms_by_id[$room_id]; + } else { + log_warning("Could not find room $room_id from listing $id."); + } + } + } + if (isset($listing_props['sogs'])) { + foreach ($listing_props['sogs'] as $public_key) { + if (isset($sogs_by_pubkey[$public_key])) { + /** @var \CommunityServer $sogs */ + $sogs = $sogs_by_pubkey[$public_key]; + array_push($rooms, ...$sogs->rooms); + } else { + log_warning("Could not find sogs $public_key from listing $id."); + } + } + } + $listings[$id] = new CommunityListing( + $listing_props['name'], + $listing_props['rating'], + $rooms + ); + } + + return $listings; + } + + function generate_listings() { + global $LISTING_PROVIDER_LISTING_SUMMARY, $LISTING_PROVIDER_LISTINGS; + log_info("Generating listings..."); + + $listings_resolved = resolve_listings_config(); + log_value($listings_resolved); + $summaries = array_map(function(\CommunityListing $listing) { + return $listing->to_summary(); + }, $listings_resolved); + file_put_contents($LISTING_PROVIDER_LISTING_SUMMARY, json_encode($summaries)); + foreach ($listings_resolved as $id => $listing) { + file_put_contents( + "$LISTING_PROVIDER_LISTINGS/$id", + json_encode($listing) + ); + } + $listings_count = count($listings_resolved); + log_info("Generated $listings_count listings."); + } + + file_exists($LISTING_PROVIDER_LISTINGS) or mkdir($LISTING_PROVIDER_LISTINGS, 0755, true); + + $options = getopt("v", ["verbose"]); + if (isset($options["v"]) or isset($options["verbose"])) { + $LOGGING_VERBOSITY = LoggingVerbosity::Debug; + } + + generate_listings(); +?> \ No newline at end of file From aea74ba7b9ab21a48ae6ef3b96f8a41f10274826 Mon Sep 17 00:00:00 2001 From: gravel Date: Thu, 1 Jun 2023 16:14:19 +0000 Subject: [PATCH 7/8] Reflect listing provider in Makefile --- Makefile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 87a57dd..4a15897 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,9 @@ list: # Refresh listing and generate HTML. sco: fetch html +# Refresh listing, generate HTML and update listing provider. +all: fetch html listing + # Fetch room listing. fetch: /bin/php php/fetch-servers.php $(FLAGS) @@ -64,19 +67,19 @@ clean: # Build everything from scratch and test functionality. test: FLAGS = --verbose -test: clean sco open server +test: clean all open server # Build everything from scratch and test functionality on LAN. test-lan: FLAGS = --verbose -test-lan: clean sco open lan-server +test-lan: clean all open lan-server # Run basic tests without launching site in browser. test-noninteractive: FLAGS = --verbose -test-noninteractive: clean sco +test-noninteractive: clean all # Run Continuous Integration tests. test-ci: FLAGS = --verbose --fast --no-color -test-ci: clean sco +test-ci: clean all # -- Aliases -- serve: server From f137899150fcdba80f5254b2e86d4538c37e90ac Mon Sep 17 00:00:00 2001 From: gravel Date: Thu, 1 Jun 2023 16:14:36 +0000 Subject: [PATCH 8/8] Run listing provider with default jobs --- php/update-listing.php | 1 + 1 file changed, 1 insertion(+) diff --git a/php/update-listing.php b/php/update-listing.php index 9cf40cd..7db661b 100644 --- a/php/update-listing.php +++ b/php/update-listing.php @@ -1,4 +1,5 @@ \ No newline at end of file