From a4efe804192d3ed5955942cf961d2640c138d0f3 Mon Sep 17 00:00:00 2001 From: jld3103 Date: Wed, 1 Nov 2023 09:37:38 +0100 Subject: [PATCH 1/3] refactor(tool,nextcloud): Use php standalone webserver Signed-off-by: jld3103 --- CONTRIBUTING.md | 3 +-- packages/nextcloud/test/helper.dart | 7 +------ packages/nextcloud/test/spreed_test.dart | 5 +---- tool/Dockerfile.dev | 13 +++++++------ tool/dev.sh | 11 ++++++++++- tool/dev/install_app_version | 4 ++-- 6 files changed, 22 insertions(+), 21 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0797d3b6..23f26748 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,8 +38,7 @@ You can read the full documentation at https://www.conventionalcommits.org. ## Tools We maintain a collection of scripts in `./tool/`. They range from setting up a local Nextcloud server (`./tool/dev.sh`) to generating assets. -Be aware that due to limitations in Apache the dev server will crash when you resize the terminal window. - + ## Monorepo For easier development we use a monorepo structure. This means that we have multiple packages in one git repository. diff --git a/packages/nextcloud/test/helper.dart b/packages/nextcloud/test/helper.dart index 73be4edb..561494cb 100644 --- a/packages/nextcloud/test/helper.dart +++ b/packages/nextcloud/test/helper.dart @@ -156,7 +156,7 @@ Future getTestClient( return client; } -Future getDockerContainer(final DockerImage image, {final bool useApache = false}) async { +Future getDockerContainer(final DockerImage image) async { late ProcessResult result; late int port; while (true) { @@ -172,11 +172,6 @@ Future getDockerContainer(final DockerImage image, {final bool '-p', '$port:80', image, - if (!useApache) ...[ - 'php', - '-S', - '0.0.0.0:80', - ], ], ); // 125 means the docker run command itself has failed which indicated the port is already used diff --git a/packages/nextcloud/test/spreed_test.dart b/packages/nextcloud/test/spreed_test.dart index ff1ee9c1..d9559464 100644 --- a/packages/nextcloud/test/spreed_test.dart +++ b/packages/nextcloud/test/spreed_test.dart @@ -18,10 +18,7 @@ void main() { late DockerContainer container; late TestNextcloudClient client1; setUp(() async { - container = await getDockerContainer( - image, - useApache: true, - ); + container = await getDockerContainer(image); client1 = await getTestClient(container); }); tearDown(() => container.destroy()); diff --git a/tool/Dockerfile.dev b/tool/Dockerfile.dev index 602893a1..d44e6d1a 100644 --- a/tool/Dockerfile.dev +++ b/tool/Dockerfile.dev @@ -1,9 +1,7 @@ -ARG SERVER_VERSION=27.1.3@sha256:4ab3159b25310ffdb699f532d45f706d5d949100ed7ac11820d83467f29678b9 +ARG SERVER_VERSION=27.1.3-fpm-alpine@sha256:70ddd4abb804bc061a4f882a81cc62a219b262c0ed1784439e66b091ef26104b FROM nextcloud:$SERVER_VERSION WORKDIR /usr/src/nextcloud -RUN chown -R www-data:www-data . -USER www-data RUN ./occ maintenance:install --admin-pass admin --admin-email admin@example.com RUN ./occ config:system:set allow_local_remote_servers --value=true @@ -30,8 +28,11 @@ RUN install_app_version spreed https://github.com/nextcloud-releases/spreed/rele RUN ./occ talk:turn:add turn,turns staticauth.openrelay.metered.ca:443 udp,tcp --secret openrelayprojectsecret RUN ./occ app:enable password_policy -RUN (sh /entrypoint.sh php -S 0.0.0.0:8080 &) && \ - until curl -s -o /dev/null http://localhost:8080/status.php; do true; done && \ +RUN (sh /entrypoint.sh php -S 0.0.0.0:80 &) && \ + until curl -s -o /dev/null http://localhost/status.php; do true; done && \ # Do not setup the demo user here - for user in admin user1 user2; do curl -u "$user:$user" -H "ocs-apirequest: true" -s -o /dev/null http://localhost:8080/ocs/v2.php/cloud/user; done + for user in admin user1 user2; do curl -u "$user:$user" -H "ocs-apirequest: true" -s -o /dev/null http://localhost/ocs/v2.php/cloud/user; done COPY --chown=www-data:www-data overlay /usr/src/nextcloud/ + +ENV PHP_CLI_SERVER_WORKERS=10 +CMD ["php", "-S", "0.0.0.0:80"] diff --git a/tool/dev.sh b/tool/dev.sh index a08d3e79..7967c2f8 100755 --- a/tool/dev.sh +++ b/tool/dev.sh @@ -6,4 +6,13 @@ source tool/common.sh ./tool/build-dev-container.sh echo "Running development instance on http://localhost. To access it in an Android Emulator use http://10.0.2.2" -docker run --rm -v nextcloud-neon-dev:/usr/src/nextcloud -v nextcloud-neon-dev:/var/www/html -p "80:80" --add-host=host.docker.internal:host-gateway "$(image_tag "dev:latest")" + +container="$(docker run -d --rm -v nextcloud-neon-dev:/usr/src/nextcloud -v nextcloud-neon-dev:/var/www/html -p "80:80" --add-host=host.docker.internal:host-gateway "$(image_tag "dev:latest")")" +function cleanup() { + docker kill "$container" +} +trap cleanup EXIT + +docker logs -f "$container" & + +sleep infinity diff --git a/tool/dev/install_app_version b/tool/dev/install_app_version index c0d7c2cb..622d23b2 100755 --- a/tool/dev/install_app_version +++ b/tool/dev/install_app_version @@ -1,5 +1,5 @@ -#!/bin/bash -set -euxo pipefail +#!/bin/sh +set -euxo app="$1" url="$2" From 0d8aeeb180b7be40b211074e01a3dfb58adb389c Mon Sep 17 00:00:00 2001 From: jld3103 Date: Wed, 1 Nov 2023 09:46:42 +0100 Subject: [PATCH 2/3] refactor(tool,nextcloud): Use dev container to serve static test files Signed-off-by: jld3103 --- cspell.json | 2 +- packages/nextcloud/test/news_test.dart | 39 +++---------------- tool/Dockerfile.dev | 2 +- .../test/files => tool/dev/static}/nasa.xml | 0 .../files => tool/dev/static}/wikipedia.xml | 0 tool/overlay/.gitkeep | 0 6 files changed, 7 insertions(+), 36 deletions(-) rename {packages/nextcloud/test/files => tool/dev/static}/nasa.xml (100%) rename {packages/nextcloud/test/files => tool/dev/static}/wikipedia.xml (100%) delete mode 100644 tool/overlay/.gitkeep diff --git a/cspell.json b/cspell.json index 7dbd722e..13b05dd0 100644 --- a/cspell.json +++ b/cspell.json @@ -13,7 +13,7 @@ "external", "packages/file_icons/lib/src/data.dart", "packages/neon_lints/lib", - "packages/nextcloud/test/files" + "tool/dev/static" ], "dictionaries": [ "bash", diff --git a/packages/nextcloud/test/news_test.dart b/packages/nextcloud/test/news_test.dart index ece64797..ef2a8ec0 100644 --- a/packages/nextcloud/test/news_test.dart +++ b/packages/nextcloud/test/news_test.dart @@ -3,7 +3,6 @@ import 'dart:async'; import 'package:nextcloud/news.dart' as news; import 'package:nextcloud/nextcloud.dart'; import 'package:test/test.dart'; -import 'package:universal_io/io.dart'; import 'helper.dart'; @@ -12,12 +11,9 @@ void main() { 'news', () { late DockerImage image; - late HttpServer rssServer; setUpAll(() async { image = await getDockerImage(); - rssServer = await getRssServer(); }); - tearDownAll(() async => rssServer.close(force: true)); late DockerContainer container; late TestNextcloudClient client; @@ -29,12 +25,12 @@ void main() { Future> addWikipediaFeed([final int? folderID]) async => client.news.addFeed( - url: 'http://host.docker.internal:${rssServer.port}/wikipedia.xml', + url: 'http://localhost/static/wikipedia.xml', folderId: folderID, ); Future> addNasaFeed() async => client.news.addFeed( - url: 'http://host.docker.internal:${rssServer.port}/nasa.xml', + url: 'http://localhost/static/nasa.xml', ); test('Is supported', () async { @@ -58,7 +54,7 @@ void main() { expect(response.body.starredCount, null); expect(response.body.newestItemId, isNotNull); expect(response.body.feeds, hasLength(1)); - expect(response.body.feeds[0].url, 'http://host.docker.internal:${rssServer.port}/wikipedia.xml'); + expect(response.body.feeds[0].url, 'http://localhost/static/wikipedia.xml'); response = await client.news.listFeeds(); expect(response.statusCode, 200); @@ -67,7 +63,7 @@ void main() { expect(response.body.starredCount, 0); expect(response.body.newestItemId, isNotNull); expect(response.body.feeds, hasLength(1)); - expect(response.body.feeds[0].url, 'http://host.docker.internal:${rssServer.port}/wikipedia.xml'); + expect(response.body.feeds[0].url, 'http://localhost/static/wikipedia.xml'); }); test('Delete feed', () async { @@ -401,7 +397,7 @@ void main() { expect(response.body.newestItemId, isNotNull); expect(response.body.feeds, hasLength(1)); expect(response.body.feeds[0].folderId, 1); - expect(response.body.feeds[0].url, 'http://host.docker.internal:${rssServer.port}/wikipedia.xml'); + expect(response.body.feeds[0].url, 'http://localhost/static/wikipedia.xml'); }); test('Mark folder as read', () async { @@ -430,28 +426,3 @@ void main() { timeout: timeout, ); } - -Future getRssServer() async { - final wikipediaRss = File('test/files/wikipedia.xml').readAsStringSync(); - final nasaRss = File('test/files/nasa.xml').readAsStringSync(); - while (true) { - try { - final port = randomPort(); - final server = await HttpServer.bind(InternetAddress.anyIPv6, port); - unawaited( - server.forEach((final request) async { - switch (request.uri.path) { - case '/wikipedia.xml': - request.response.write(wikipediaRss); - case '/nasa.xml': - request.response.write(nasaRss); - default: - request.response.statusCode = HttpStatus.badRequest; - } - await request.response.close(); - }), - ); - return server; - } catch (_) {} - } -} diff --git a/tool/Dockerfile.dev b/tool/Dockerfile.dev index d44e6d1a..7f818d19 100644 --- a/tool/Dockerfile.dev +++ b/tool/Dockerfile.dev @@ -32,7 +32,7 @@ RUN (sh /entrypoint.sh php -S 0.0.0.0:80 &) && \ until curl -s -o /dev/null http://localhost/status.php; do true; done && \ # Do not setup the demo user here for user in admin user1 user2; do curl -u "$user:$user" -H "ocs-apirequest: true" -s -o /dev/null http://localhost/ocs/v2.php/cloud/user; done -COPY --chown=www-data:www-data overlay /usr/src/nextcloud/ +COPY --chown=www-data:www-data dev/static /usr/src/nextcloud/static ENV PHP_CLI_SERVER_WORKERS=10 CMD ["php", "-S", "0.0.0.0:80"] diff --git a/packages/nextcloud/test/files/nasa.xml b/tool/dev/static/nasa.xml similarity index 100% rename from packages/nextcloud/test/files/nasa.xml rename to tool/dev/static/nasa.xml diff --git a/packages/nextcloud/test/files/wikipedia.xml b/tool/dev/static/wikipedia.xml similarity index 100% rename from packages/nextcloud/test/files/wikipedia.xml rename to tool/dev/static/wikipedia.xml diff --git a/tool/overlay/.gitkeep b/tool/overlay/.gitkeep deleted file mode 100644 index e69de29b..00000000 From eefcdf810faff6b43abee5f0970cc84eae18ea0f Mon Sep 17 00:00:00 2001 From: jld3103 Date: Wed, 1 Nov 2023 10:03:15 +0100 Subject: [PATCH 3/3] refactor(nextcloud): Rely on prebuilt docker image for testing Signed-off-by: jld3103 --- packages/nextcloud/test/core_test.dart | 5 +- packages/nextcloud/test/dashboard_test.dart | 5 +- packages/nextcloud/test/helper.dart | 51 +++++++------------ packages/nextcloud/test/news_test.dart | 7 +-- packages/nextcloud/test/notes_test.dart | 5 +- .../nextcloud/test/notifications_test.dart | 10 +--- .../nextcloud/test/provisioning_api_test.dart | 5 +- packages/nextcloud/test/settings_test.dart | 47 ++++++++--------- packages/nextcloud/test/spreed_test.dart | 5 +- packages/nextcloud/test/uppush_test.dart | 5 +- packages/nextcloud/test/user_status_test.dart | 5 +- packages/nextcloud/test/webdav_test.dart | 5 +- 12 files changed, 54 insertions(+), 101 deletions(-) diff --git a/packages/nextcloud/test/core_test.dart b/packages/nextcloud/test/core_test.dart index 0c874beb..dbe9954a 100644 --- a/packages/nextcloud/test/core_test.dart +++ b/packages/nextcloud/test/core_test.dart @@ -8,13 +8,10 @@ void main() { group( 'core', () { - late DockerImage image; - setUpAll(() async => image = await getDockerImage()); - late DockerContainer container; late TestNextcloudClient client; setUp(() async { - container = await getDockerContainer(image); + container = await getDockerContainer(); client = await getTestClient(container); }); tearDown(() => container.destroy()); diff --git a/packages/nextcloud/test/dashboard_test.dart b/packages/nextcloud/test/dashboard_test.dart index ca18058a..97ba4f1a 100644 --- a/packages/nextcloud/test/dashboard_test.dart +++ b/packages/nextcloud/test/dashboard_test.dart @@ -7,13 +7,10 @@ void main() { group( 'dashboard', () { - late DockerImage image; - setUpAll(() async => image = await getDockerImage()); - late DockerContainer container; late TestNextcloudClient client; setUp(() async { - container = await getDockerContainer(image); + container = await getDockerContainer(); client = await getTestClient(container); }); tearDown(() => container.destroy()); diff --git a/packages/nextcloud/test/helper.dart b/packages/nextcloud/test/helper.dart index 561494cb..20c93025 100644 --- a/packages/nextcloud/test/helper.dart +++ b/packages/nextcloud/test/helper.dart @@ -156,8 +156,24 @@ Future getTestClient( return client; } -Future getDockerContainer(final DockerImage image) async { - late ProcessResult result; +Future getDockerContainer() async { + const dockerImageName = 'ghcr.io/nextcloud/neon/dev'; + + var result = await runExecutableArguments( + 'docker', + [ + 'images', + '-q', + dockerImageName, + ], + ); + if (result.exitCode != 0) { + throw Exception('Querying docker image failed: ${result.stderr}'); + } + if (result.stdout.toString().isEmpty) { + throw Exception('Missing docker image $dockerImageName. Please build it using ./tool/build-dev-container.sh'); + } + late int port; while (true) { port = randomPort(); @@ -171,7 +187,7 @@ Future getDockerContainer(final DockerImage image) async { 'host.docker.internal:host-gateway', '-p', '$port:80', - image, + dockerImageName, ], ); // 125 means the docker run command itself has failed which indicated the port is already used @@ -190,35 +206,6 @@ Future getDockerContainer(final DockerImage image) async { ); } -typedef DockerImage = String; - -Future getDockerImage() async { - const dockerImageName = 'nextcloud-neon-dev'; - - final inputStream = StreamController>(); - final process = runExecutableArguments( - 'docker', - [ - 'build', - '-t', - dockerImageName, - '-f', - '-', - '../../tool', - ], - stdin: inputStream.stream, - ); - inputStream.add(utf8.encode(File('../../tool/Dockerfile.dev').readAsStringSync())); - await inputStream.close(); - - final result = await process; - if (result.exitCode != 0) { - throw Exception('Failed to build docker image'); - } - - return dockerImageName; -} - class TestNextcloudUser { TestNextcloudUser( this.username, diff --git a/packages/nextcloud/test/news_test.dart b/packages/nextcloud/test/news_test.dart index ef2a8ec0..b3ef3fd1 100644 --- a/packages/nextcloud/test/news_test.dart +++ b/packages/nextcloud/test/news_test.dart @@ -10,15 +10,10 @@ void main() { group( 'news', () { - late DockerImage image; - setUpAll(() async { - image = await getDockerImage(); - }); - late DockerContainer container; late TestNextcloudClient client; setUp(() async { - container = await getDockerContainer(image); + container = await getDockerContainer(); client = await getTestClient(container); }); tearDown(() => container.destroy()); diff --git a/packages/nextcloud/test/notes_test.dart b/packages/nextcloud/test/notes_test.dart index 1d49683e..bc80c6fb 100644 --- a/packages/nextcloud/test/notes_test.dart +++ b/packages/nextcloud/test/notes_test.dart @@ -9,13 +9,10 @@ void main() { group( 'notes', () { - late DockerImage image; - setUpAll(() async => image = await getDockerImage()); - late DockerContainer container; late TestNextcloudClient client; setUp(() async { - container = await getDockerContainer(image); + container = await getDockerContainer(); client = await getTestClient(container); }); tearDown(() => container.destroy()); diff --git a/packages/nextcloud/test/notifications_test.dart b/packages/nextcloud/test/notifications_test.dart index 093423e3..0520bf84 100644 --- a/packages/nextcloud/test/notifications_test.dart +++ b/packages/nextcloud/test/notifications_test.dart @@ -7,13 +7,10 @@ import 'helper.dart'; void main() { group('notifications', () { - late DockerImage image; - setUpAll(() async => image = await getDockerImage()); - late DockerContainer container; late TestNextcloudClient client; setUp(() async { - container = await getDockerContainer(image); + container = await getDockerContainer(); client = await getTestClient( container, username: 'admin', @@ -103,13 +100,10 @@ void main() { }); group('Push', () { - late DockerImage image; - setUpAll(() async => image = await getDockerImage()); - late DockerContainer container; late TestNextcloudClient client; setUp(() async { - container = await getDockerContainer(image); + container = await getDockerContainer(); client = await getTestClient( container, username: 'admin', diff --git a/packages/nextcloud/test/provisioning_api_test.dart b/packages/nextcloud/test/provisioning_api_test.dart index 067b4446..6f175ced 100644 --- a/packages/nextcloud/test/provisioning_api_test.dart +++ b/packages/nextcloud/test/provisioning_api_test.dart @@ -7,13 +7,10 @@ void main() { group( 'provisioning_api', () { - late DockerImage image; - setUpAll(() async => image = await getDockerImage()); - late DockerContainer container; late TestNextcloudClient client; setUp(() async { - container = await getDockerContainer(image); + container = await getDockerContainer(); client = await getTestClient( container, username: 'admin', diff --git a/packages/nextcloud/test/settings_test.dart b/packages/nextcloud/test/settings_test.dart index 4a60ef5c..9ad42508 100644 --- a/packages/nextcloud/test/settings_test.dart +++ b/packages/nextcloud/test/settings_test.dart @@ -5,29 +5,30 @@ import 'package:test/test.dart'; import 'helper.dart'; -Future main() async { - await run(await getDockerImage()); -} - -Future run(final DockerImage image) async { - group('settings', () { - late DockerContainer container; - late TestNextcloudClient client; - setUp(() async { - container = await getDockerContainer(image); - client = await getTestClient( - container, - username: 'admin', - ); - }); - tearDown(() => container.destroy()); +void main() { + group( + 'settings', + () { + late DockerContainer container; + late TestNextcloudClient client; + setUp(() async { + container = await getDockerContainer(); + client = await getTestClient( + container, + username: 'admin', + ); + }); + tearDown(() => container.destroy()); - group('Logs', () { - test('Download', () async { - final response = await client.settings.logSettings.download(); - final logs = utf8.decode(response.body); - expect(logs, await container.nextcloudLogs()); + group('Logs', () { + test('Download', () async { + final response = await client.settings.logSettings.download(); + final logs = utf8.decode(response.body); + expect(logs, await container.nextcloudLogs()); + }); }); - }); - }); + }, + retry: retryCount, + timeout: timeout, + ); } diff --git a/packages/nextcloud/test/spreed_test.dart b/packages/nextcloud/test/spreed_test.dart index d9559464..2b88772e 100644 --- a/packages/nextcloud/test/spreed_test.dart +++ b/packages/nextcloud/test/spreed_test.dart @@ -12,13 +12,10 @@ void main() { group( 'spreed', () { - late DockerImage image; - setUpAll(() async => image = await getDockerImage()); - late DockerContainer container; late TestNextcloudClient client1; setUp(() async { - container = await getDockerContainer(image); + container = await getDockerContainer(); client1 = await getTestClient(container); }); tearDown(() => container.destroy()); diff --git a/packages/nextcloud/test/uppush_test.dart b/packages/nextcloud/test/uppush_test.dart index dd93f61f..2ecd3e7f 100644 --- a/packages/nextcloud/test/uppush_test.dart +++ b/packages/nextcloud/test/uppush_test.dart @@ -7,13 +7,10 @@ void main() { group( 'uppush', () { - late DockerImage image; - setUpAll(() async => image = await getDockerImage()); - late DockerContainer container; late TestNextcloudClient client; setUp(() async { - container = await getDockerContainer(image); + container = await getDockerContainer(); client = await getTestClient( container, username: 'admin', diff --git a/packages/nextcloud/test/user_status_test.dart b/packages/nextcloud/test/user_status_test.dart index 279442dc..b53905f9 100644 --- a/packages/nextcloud/test/user_status_test.dart +++ b/packages/nextcloud/test/user_status_test.dart @@ -7,13 +7,10 @@ void main() { group( 'user_status', () { - late DockerImage image; - setUpAll(() async => image = await getDockerImage()); - late DockerContainer container; late TestNextcloudClient client; setUp(() async { - container = await getDockerContainer(image); + container = await getDockerContainer(); client = await getTestClient(container); }); tearDown(() => container.destroy()); diff --git a/packages/nextcloud/test/webdav_test.dart b/packages/nextcloud/test/webdav_test.dart index bfc3eeea..523ddc40 100644 --- a/packages/nextcloud/test/webdav_test.dart +++ b/packages/nextcloud/test/webdav_test.dart @@ -26,14 +26,11 @@ void main() { group( 'webdav', () { - late DockerImage image; - setUpAll(() async => image = await getDockerImage()); - late DockerContainer container; late TestNextcloudClient client; setUp(() async { - container = await getDockerContainer(image); + container = await getDockerContainer(); client = await getTestClient(container); }); tearDown(() => container.destroy());