From ede71e4331df1eebb266ab342d5d5e5857943fc1 Mon Sep 17 00:00:00 2001 From: Renoir Boulanger Date: Wed, 15 Apr 2015 18:30:55 -0400 Subject: [PATCH 01/18] And so you run in Docker --- Dockerfile | 32 ++++++++++++++++++++++++++++++++ Makefile | 10 ++++++++++ README.md | 18 ++++++++++++++++-- bin/run.sh | 8 ++++++++ package.json | 4 +++- 5 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 bin/run.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..84af322 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +# +# Publican Docker runner +# +# See also: +# * https://github.com/nodesource/docker-node/blob/master/ubuntu/trusty/node/0.10.36/Dockerfile + +MAINTAINER renoirb 'https://github.com/renoirb' + +FROM nodesource/trusty:0.10.36 + +RUN mkdir -p /srv/webplatform/specs/queue +RUN mkdir -p /srv/webplatform/specs/data/logs + +RUN /usr/sbin/useradd --home-dir /srv/webplatform/specs --shell /bin/bash nonroot +RUN /usr/sbin/adduser nonroot sudo + +COPY . /srv/webplatform/specs/ + +RUN chown -R nonroot /usr/local/ +RUN chown -R nonroot /usr/lib/ +RUN chown -R nonroot /usr/bin/ +RUN chown -R nonroot /srv/webplatform/specs/ + +ADD package.json /tmp/package.json +RUN cd /tmp && npm install +RUN cp -a /tmp/node_modules /srv/webplatform/specs + +USER nonroot +WORKDIR /srv/webplatform/specs + +ENTRYPOINT ["/bin/bash", "/srv/webplatform/specs/bin/run.sh"] + diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3a255ea --- /dev/null +++ b/Makefile @@ -0,0 +1,10 @@ +SHELL := bash + +default: run + +build: + docker build -t webspecs/publican . + +run: + docker run -it --rm -v "$(CURDIR)/data":/srv/webplatform/specs/data webspecs/publican + diff --git a/README.md b/README.md index 6e62926..c246131 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,18 @@ -publican -======== +# Publican The set of tools used to automate publishing in the WebSpecs project + +## Run in Docker + +You can optionnaly run the script within a [Docker container](http://docker.io/) that acts as an isolated job runner. + +The following takes into account that you can run `docker` commands. + +```bash +git checkout task-based +make build +make +``` + +Please refer to the appropriate documentation if you want to [install](https://docs.docker.com/installation/#installation) and run this project from a Docker container. + diff --git a/bin/run.sh b/bin/run.sh new file mode 100644 index 0000000..db56782 --- /dev/null +++ b/bin/run.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +export RUNDIR="/srv/webplatform/specs" + +cd $RUNDIR + +node_modules/forever/bin/forever start $RUNDIR/bin/server.js +node_modules/forever/bin/forever --fifo logs 0 diff --git a/package.json b/package.json index 61316d2..2ce4e25 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,7 @@ { "name": "publican", - "version": "0.3.0", + "description": "The set of tools used to automate publishing in the WebSpecs project", + "version": "0.3.1", "license": "MIT", "dependencies": { "async": "0.9.0", @@ -18,6 +19,7 @@ "tmp": "0.0.25", "whacko": "0.17.4", "winston": "0.9.0", + "forever": "~0.14", "winston-mail": "0.3.2" }, "bin": { From e91d602ad210199ae18adcf79320a6aa63edb84d Mon Sep 17 00:00:00 2001 From: Renoir Boulanger Date: Wed, 15 Apr 2015 18:51:14 -0400 Subject: [PATCH 02/18] WIP --- Dockerfile | 6 +++--- Makefile | 4 +++- bin/run.sh | 1 - data/config.json.dist | 20 ++++++++++++++++++++ 4 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 data/config.json.dist diff --git a/Dockerfile b/Dockerfile index 84af322..db21695 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,8 +4,6 @@ # See also: # * https://github.com/nodesource/docker-node/blob/master/ubuntu/trusty/node/0.10.36/Dockerfile -MAINTAINER renoirb 'https://github.com/renoirb' - FROM nodesource/trusty:0.10.36 RUN mkdir -p /srv/webplatform/specs/queue @@ -25,8 +23,10 @@ ADD package.json /tmp/package.json RUN cd /tmp && npm install RUN cp -a /tmp/node_modules /srv/webplatform/specs +EXPOSE 7002 + USER nonroot WORKDIR /srv/webplatform/specs ENTRYPOINT ["/bin/bash", "/srv/webplatform/specs/bin/run.sh"] - +CMD ["node_modules/forever/bin/forever", "--fifo", "logs", "0"] diff --git a/Makefile b/Makefile index 3a255ea..d371037 100644 --- a/Makefile +++ b/Makefile @@ -6,5 +6,7 @@ build: docker build -t webspecs/publican . run: - docker run -it --rm -v "$(CURDIR)/data":/srv/webplatform/specs/data webspecs/publican + docker run -it --rm -v "$(CURDIR)/data":/srv/webplatform/specs/data -p 7002 webspecs/publican +tail: + tail -f data/logs/all.log diff --git a/bin/run.sh b/bin/run.sh index db56782..ef206b5 100644 --- a/bin/run.sh +++ b/bin/run.sh @@ -5,4 +5,3 @@ export RUNDIR="/srv/webplatform/specs" cd $RUNDIR node_modules/forever/bin/forever start $RUNDIR/bin/server.js -node_modules/forever/bin/forever --fifo logs 0 diff --git a/data/config.json.dist b/data/config.json.dist new file mode 100644 index 0000000..ed7c3e5 --- /dev/null +++ b/data/config.json.dist @@ -0,0 +1,20 @@ +{ + "rsyncRemote": "ubuntu@specs.production.wpdn" +, "rsyncPath": "/srv/webplatform/specs/" +, "python": "python2" +, "logFile": "logs/all.log" +, "email": { + "to": "robin@berjon.com" + , "from": "publican-noreply@webplatform.org" + , "host": "mail.messagingengine.com" + , "port": 465 + , "username": "robin@berjon.com" + , "password": "XXXX not the real thing, will not work. Should be replaced by a login that can email from here." + , "ssl": true + , "tls": false + , "level": "error" + , "handleExceptions": true + } +, "purgeAllURL": "https://api.fastly.com/service/not_the_real_thing/purge_all" +, "purgeAllKey": "not_the_real_thing" +} From 3ff84b3c161eebf834f634d9863853a959a81a82 Mon Sep 17 00:00:00 2001 From: Renoir Boulanger Date: Wed, 15 Apr 2015 19:29:29 -0400 Subject: [PATCH 03/18] This was OK, reverting --- Dockerfile | 2 +- bin/run.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index db21695..8ca2d66 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,4 +29,4 @@ USER nonroot WORKDIR /srv/webplatform/specs ENTRYPOINT ["/bin/bash", "/srv/webplatform/specs/bin/run.sh"] -CMD ["node_modules/forever/bin/forever", "--fifo", "logs", "0"] +#CMD ["node_modules/forever/bin/forever", "--fifo", "logs", "0"] diff --git a/bin/run.sh b/bin/run.sh index ef206b5..db56782 100644 --- a/bin/run.sh +++ b/bin/run.sh @@ -5,3 +5,4 @@ export RUNDIR="/srv/webplatform/specs" cd $RUNDIR node_modules/forever/bin/forever start $RUNDIR/bin/server.js +node_modules/forever/bin/forever --fifo logs 0 From 388529d03d001a900f322667e0e4edb8224309e8 Mon Sep 17 00:00:00 2001 From: Renoir Boulanger Date: Mon, 20 Apr 2015 13:00:32 -0400 Subject: [PATCH 04/18] Added moar instructions 4 Docker --- README.md | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c246131..db16de8 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,26 @@ The set of tools used to automate publishing in the WebSpecs project + + ## Run in Docker -You can optionnaly run the script within a [Docker container](http://docker.io/) that acts as an isolated job runner. +If you have [Docker installed](https://docs.docker.com/installation/) and running, you can run it in a few commands. + +```bash +docker pull webspecs/publican +docker run -it --rm -p 7002:7002 webspecs/publican +``` + +If you want to run with your own customizations, you’d have to do also have [**Docker Compose** installed](https://docs.docker.com/compose/install/) and do the following. + +Follow the instructions in **Run using _Docker Compose_** in [the registry description instructions](https://registry.hub.docker.com/u/webspecs/publican/) + + + +## Build Docker container + +If you changed project source, you’ll need to build another docker container. The following takes into account that you can run `docker` commands. @@ -16,3 +33,29 @@ make Please refer to the appropriate documentation if you want to [install](https://docs.docker.com/installation/#installation) and run this project from a Docker container. + + +## Push Docker container to Docker Hub registry + +We can quickly run a compiled container from the Docker Hub which [acts in a similar way as **git** would](http://docs.docker.com/reference/commandline/cli/#commit-a-container). + +This project Docker registry entry is at [registry.hub.docker.com/u/**webspecs/publican**](https://registry.hub.docker.com/u/webspecs/publican/). + +To do so, run a container, commit and push. More detailed instructions in [Docker docs at the *commandline* section](http://docs.docker.com/reference/commandline/cli/#commit-a-container) + +Get current running container; that’s what we commit. + +```bash +docker ps + +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +3cce48d3b30c servicepublican_publican:latest "/bin/bash /srv/webp 11 minutes ago Up 11 minutes 0.0.0.0:8002->7002/tcp sandboxpublican_web_1" +``` + +If you have write access as a *collaborator* to the *webspecs* organization within Docker hub; Commit and push + +```bash +docker commit 3cce48d3b30c +docker push webspecs/publican +``` + From 7b902d4d7ab96e58b2b51751975fe727ea0e2d76 Mon Sep 17 00:00:00 2001 From: Renoir Boulanger Date: Mon, 20 Apr 2015 15:01:07 -0400 Subject: [PATCH 05/18] Documentation and refactor of the container --- DOCKER.md | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ Dockerfile | 29 ++++++++++++------ Makefile | 6 +++- README.md | 57 ---------------------------------- 4 files changed, 114 insertions(+), 68 deletions(-) create mode 100644 DOCKER.md diff --git a/DOCKER.md b/DOCKER.md new file mode 100644 index 0000000..ec34f13 --- /dev/null +++ b/DOCKER.md @@ -0,0 +1,90 @@ +# Publican + +The set of tools used to automate publishing in the WebSpecs project + + + +## Run in Docker + +If you have [Docker installed](https://docs.docker.com/installation/) and running, you can run it in a few commands. + +```bash +docker pull webspecs/publican +docker run -it --rm -p 7002:7002 webspecs/publican +``` + +If you want to run with your own customizations, you’d have to do also have [**Docker Compose** installed](https://docs.docker.com/compose/install/) and do the following. + +Follow the instructions in **Run using _Docker Compose_** in [the registry description instructions](https://registry.hub.docker.com/u/webspecs/publican/) + + + +## Build Docker container + +If you changed project source, you’ll need to build another docker container. + +The following takes into account that you can run `docker` commands. + +```bash +git checkout task-based +make build +make +``` + +Which extends into something similar to: + +``` +docker run -it --rm -v /Users/renoirb/workspaces/webplatform/service-publican/repo/data:/srv/webplatform/specs/data -p 8002:7002 webspecs/publican bin/run.sh +``` + +Refer to the appropriate documentation if you want to [install](https://docs.docker.com/installation/#installation) and run this project from a Docker container. + + + +## Enter the container shell + +This is useful to understand what’s happening + +```bash +make bash +``` + + + +## Push Docker container to Docker Hub registry + +The Docker registry is basically a storage service of docker container states from which we can pull and run from it. +It saves us to rebuild the container on every server that would run the container. + +Docker uses a semantic similar to git, a container can be commited and pushed to a repository. That’s what we’ll do here. + +Rever to [Docker command line "commit" documentation](http://docs.docker.com/reference/commandline/cli/#commit-a-container) for further details. + +This project Docker registry entry is at [registry.hub.docker.com/u/**webspecs/publican**](https://registry.hub.docker.com/u/webspecs/publican/). + +To commit a container, we need a running one. Once you have one, you can commit. + +Get current running container; that’s what we commit. + +```bash +docker ps + +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +3cce48d3b30c servicepublican_publican:latest "/bin/bash /srv/webp 11 minutes ago Up 11 minutes 0.0.0.0:8002->7002/tcp sandboxpublican_web_1" +``` + +If you have write access as a *collaborator* to the *webspecs* organization within Docker hub; Commit and push + +```bash +docker commit 3cce48d3b30c +docker push webspecs/publican +``` + +Done. + + + +## Run from Docker composer + +TODO, see notes in [**webspecs/publican**](https://registry.hub.docker.com/u/webspecs/publican/) Docker hub description. + diff --git a/Dockerfile b/Dockerfile index 8ca2d66..ca3d80d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,27 +6,36 @@ FROM nodesource/trusty:0.10.36 +MAINTAINER Renoir Boulanger + RUN mkdir -p /srv/webplatform/specs/queue RUN mkdir -p /srv/webplatform/specs/data/logs +RUN apt-get update \ + && apt-get install -y --force-yes git + RUN /usr/sbin/useradd --home-dir /srv/webplatform/specs --shell /bin/bash nonroot RUN /usr/sbin/adduser nonroot sudo COPY . /srv/webplatform/specs/ -RUN chown -R nonroot /usr/local/ -RUN chown -R nonroot /usr/lib/ -RUN chown -R nonroot /usr/bin/ -RUN chown -R nonroot /srv/webplatform/specs/ +RUN chown -R nonroot:nonroot /srv/webplatform/specs/ + +USER nonroot +WORKDIR /srv/webplatform/specs -ADD package.json /tmp/package.json -RUN cd /tmp && npm install -RUN cp -a /tmp/node_modules /srv/webplatform/specs +ENV HOME /srv/webplatform/specs +ENV GIT_DISCOVERY_ACROSS_FILESYSTEM true + +RUN npm install EXPOSE 7002 -USER nonroot -WORKDIR /srv/webplatform/specs +# Allow cli entry for debug, but make sure docker-compose.yml uses "command: bin/run.sh" +ENTRYPOINT ["/bin/bash"] + +# Note leftover: Ideally, it should exclusively run +#ENTRYPOINT ["/bin/bash", "/srv/webplatform/specs/bin/run.sh"] -ENTRYPOINT ["/bin/bash", "/srv/webplatform/specs/bin/run.sh"] +# Note leftover: What it ends up doing #CMD ["node_modules/forever/bin/forever", "--fifo", "logs", "0"] diff --git a/Makefile b/Makefile index d371037..c84fc75 100644 --- a/Makefile +++ b/Makefile @@ -5,8 +5,12 @@ default: run build: docker build -t webspecs/publican . -run: +bash: docker run -it --rm -v "$(CURDIR)/data":/srv/webplatform/specs/data -p 7002 webspecs/publican +run: + docker run -it --rm -v "$(CURDIR)/data":/srv/webplatform/specs/data -p 7002 webspecs/publican bin/run.sh + docker ps | grep publican + tail: tail -f data/logs/all.log diff --git a/README.md b/README.md index db16de8..2812b95 100644 --- a/README.md +++ b/README.md @@ -2,60 +2,3 @@ The set of tools used to automate publishing in the WebSpecs project - - -## Run in Docker - -If you have [Docker installed](https://docs.docker.com/installation/) and running, you can run it in a few commands. - -```bash -docker pull webspecs/publican -docker run -it --rm -p 7002:7002 webspecs/publican -``` - -If you want to run with your own customizations, you’d have to do also have [**Docker Compose** installed](https://docs.docker.com/compose/install/) and do the following. - -Follow the instructions in **Run using _Docker Compose_** in [the registry description instructions](https://registry.hub.docker.com/u/webspecs/publican/) - - - -## Build Docker container - -If you changed project source, you’ll need to build another docker container. - -The following takes into account that you can run `docker` commands. - -```bash -git checkout task-based -make build -make -``` - -Please refer to the appropriate documentation if you want to [install](https://docs.docker.com/installation/#installation) and run this project from a Docker container. - - - -## Push Docker container to Docker Hub registry - -We can quickly run a compiled container from the Docker Hub which [acts in a similar way as **git** would](http://docs.docker.com/reference/commandline/cli/#commit-a-container). - -This project Docker registry entry is at [registry.hub.docker.com/u/**webspecs/publican**](https://registry.hub.docker.com/u/webspecs/publican/). - -To do so, run a container, commit and push. More detailed instructions in [Docker docs at the *commandline* section](http://docs.docker.com/reference/commandline/cli/#commit-a-container) - -Get current running container; that’s what we commit. - -```bash -docker ps - -CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -3cce48d3b30c servicepublican_publican:latest "/bin/bash /srv/webp 11 minutes ago Up 11 minutes 0.0.0.0:8002->7002/tcp sandboxpublican_web_1" -``` - -If you have write access as a *collaborator* to the *webspecs* organization within Docker hub; Commit and push - -```bash -docker commit 3cce48d3b30c -docker push webspecs/publican -``` - From 903af3a91f224cee4408682b3bb0cb7158c3c78e Mon Sep 17 00:00:00 2001 From: Renoir Boulanger Date: Mon, 20 Apr 2015 15:08:14 -0400 Subject: [PATCH 06/18] Forgotten to adjust bin/run.sh in example --- DOCKER.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DOCKER.md b/DOCKER.md index ec34f13..4d913a1 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -10,7 +10,7 @@ If you have [Docker installed](https://docs.docker.com/installation/) and runnin ```bash docker pull webspecs/publican -docker run -it --rm -p 7002:7002 webspecs/publican +docker run -it --rm -p 7002:7002 webspecs/publican bin/run.sh ``` If you want to run with your own customizations, you’d have to do also have [**Docker Compose** installed](https://docs.docker.com/compose/install/) and do the following. From a04ca0f88b5073cced88b508994bb99c1ec06270 Mon Sep 17 00:00:00 2001 From: Renoir Boulanger Date: Wed, 6 May 2015 10:17:18 -0400 Subject: [PATCH 07/18] webspecs/publican#20 proposal; Otherwise it breaks while executing hook to build without SSH keys --- lib/manager.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/manager.js b/lib/manager.js index 0b2f33a..fd45b17 100644 --- a/lib/manager.js +++ b/lib/manager.js @@ -37,7 +37,7 @@ Manager.prototype = { , updateContextForRepo: function (ctx, opt) { if (opt.repository) { // git details - ctx.gitRepoURL = "git@github.com:" + opt.repository + ".git"; + ctx.gitRepoURL = "https://github.com/" + opt.repository + ".git"; ctx.gitDir = jn(dataDir, "gits", opt.repository); ctx.repository = opt.repository; } From 99f880e48cde54f52e7094358c2bf574d3107ce1 Mon Sep 17 00:00:00 2001 From: Renoir Boulanger Date: Wed, 6 May 2015 11:02:35 -0400 Subject: [PATCH 08/18] webspecs/publican#19 disable rsync (needs discussion) --- lib/tasks/init-the-index.js | 3 +-- lib/tasks/process-spec.js | 1 - lib/tasks/process-static.js | 3 +-- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/tasks/init-the-index.js b/lib/tasks/init-the-index.js index aaa2316..b4a9398 100644 --- a/lib/tasks/init-the-index.js +++ b/lib/tasks/init-the-index.js @@ -18,7 +18,6 @@ module.exports = [ ctx.subDirOnly = "index.html"; cb(); } - , require("./rsync") , require("./purge-all") ] -; +; diff --git a/lib/tasks/process-spec.js b/lib/tasks/process-spec.js index 3516fc7..9a99f94 100644 --- a/lib/tasks/process-spec.js +++ b/lib/tasks/process-spec.js @@ -28,7 +28,6 @@ module.exports = [ return this.runTask(require("./generate-bikeshed"), ctx, cb); cb(new Error("Failed to find a specification file in " + ctx.specDir)); } - , require("./rsync") , require("./purge-all") ] ; diff --git a/lib/tasks/process-static.js b/lib/tasks/process-static.js index 52deb41..8daeaea 100644 --- a/lib/tasks/process-static.js +++ b/lib/tasks/process-static.js @@ -2,7 +2,6 @@ module.exports = [ require("./git-clone-or-fetch") , require("./git-publish") - , require("./rsync") , require("./purge-all") ] -; +; From b7dbbe2f29038d87695062848565687aae530537 Mon Sep 17 00:00:00 2001 From: Renoir Boulanger Date: Wed, 6 May 2015 11:04:32 -0400 Subject: [PATCH 09/18] webspecs/publican#19 Environment runtime setup Build steps, to support either native through Makefile, and/or Dockerfile --- .gitignore | 3 +++ Dockerfile | 24 ++++++++++++++++-------- Makefile | 15 ++++++++++++--- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 8314e37..c2f6c29 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,9 @@ coverage # Compiled binary addons (http://nodejs.org/api/addons.html) build/Release +# Publican runner +publican + # Dependency directory # Commenting this out is preferred by some people, see # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git- diff --git a/Dockerfile b/Dockerfile index ca3d80d..fd2d41e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,26 +8,34 @@ FROM nodesource/trusty:0.10.36 MAINTAINER Renoir Boulanger -RUN mkdir -p /srv/webplatform/specs/queue -RUN mkdir -p /srv/webplatform/specs/data/logs +RUN mkdir -p /srv/webplatform/specs/data/queue && \ + mkdir -p /srv/webplatform/specs/data/logs && \ + mkdir -p /srv/webplatform/specs/bikeshed -RUN apt-get update \ - && apt-get install -y --force-yes git +RUN apt-get update && \ + apt-get install -yqq git python2.7 python-dev python-pip libxslt1-dev libxml2-dev zlib1g-dev && \ + pip install lxml -RUN /usr/sbin/useradd --home-dir /srv/webplatform/specs --shell /bin/bash nonroot -RUN /usr/sbin/adduser nonroot sudo +RUN /usr/sbin/useradd --home-dir /srv/webplatform/specs --shell /bin/bash nonroot && \ + /usr/sbin/adduser nonroot sudo COPY . /srv/webplatform/specs/ -RUN chown -R nonroot:nonroot /srv/webplatform/specs/ +RUN chown -R nonroot:nonroot /srv/webplatform/specs USER nonroot + +RUN git clone -b webspecs https://github.com/webspecs/bikeshed.git /srv/webplatform/specs/bikeshed && \ + pip install --user --editable /srv/webplatform/specs/bikeshed + WORKDIR /srv/webplatform/specs +ENV PATH /srv/webplatform/specs/bin:/srv/webplatform/specs/.local/bin:$PATH ENV HOME /srv/webplatform/specs ENV GIT_DISCOVERY_ACROSS_FILESYSTEM true -RUN npm install +RUN chmod +x bin/run.sh && \ + npm install EXPOSE 7002 diff --git a/Makefile b/Makefile index c84fc75..7332305 100644 --- a/Makefile +++ b/Makefile @@ -2,15 +2,24 @@ SHELL := bash default: run -build: +deps: + mkdir -p bikeshed + git clone -b webspecs https://github.com/webspecs/bikeshed.git bikeshed + pip install --user --editable bikeshed + +docker-build: docker build -t webspecs/publican . -bash: +docker-bash: docker run -it --rm -v "$(CURDIR)/data":/srv/webplatform/specs/data -p 7002 webspecs/publican -run: +docker-run: + mkdir -p $(CURDIR)/data/{logs,publish,queue} docker run -it --rm -v "$(CURDIR)/data":/srv/webplatform/specs/data -p 7002 webspecs/publican bin/run.sh docker ps | grep publican tail: tail -f data/logs/all.log + +run: docker-run tail + From 11ecf1115e2fac094ece03e78eae3a53dfbc0c89 Mon Sep 17 00:00:00 2001 From: Renoir Boulanger Date: Wed, 6 May 2015 11:27:56 -0400 Subject: [PATCH 10/18] Docker container setup with volumes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Should run bikeshed, phantomjs * Removed use of rsync, it’ll sync to data/publish/ directory * Renamed anonymous js functions to help debugging * npmrc to contextualize node_modules, nothing global plz! * .gitignore to ignore bikeshed clone (either within Docker or not) * Change location of bikeshed directory *outside* of data Includes work on: * webspecs/publican#19 Forgot npm install in make deps * webplatform/ops#153 Signed-off-by: Renoir Boulanger --- .gitignore | 4 +-- .npmrc | 1 + DOCKER.md | 10 +++++-- Dockerfile | 53 +++++++++++++++++++++------------ Makefile | 23 +++++++------- data/config.json.dist | 2 +- lib/lock.js | 10 +++---- lib/manager.js | 24 +++++++-------- lib/queue.js | 14 ++++----- lib/tasks/extract-the-index.js | 4 +-- lib/tasks/generate-bikeshed.js | 6 ++-- lib/tasks/generate-respec.js | 12 ++++---- lib/tasks/generate-the-index.js | 14 ++++----- lib/tasks/git-clone-or-fetch.js | 4 +-- lib/tasks/git-clone.js | 6 ++-- lib/tasks/git-fetch.js | 4 +-- lib/tasks/git-publish.js | 10 +++---- lib/tasks/init-all.js | 8 ++--- lib/tasks/init-bikeshed.js | 4 +-- lib/tasks/init-the-index.js | 1 - lib/tasks/process-spec.js | 1 - lib/tasks/process-specs.js | 6 ++-- lib/tasks/process-static.js | 1 - lib/tasks/process-the-index.js | 8 ++--- 24 files changed, 123 insertions(+), 107 deletions(-) create mode 100644 .npmrc diff --git a/.gitignore b/.gitignore index c2f6c29..4d693eb 100644 --- a/.gitignore +++ b/.gitignore @@ -19,8 +19,8 @@ coverage # Compiled binary addons (http://nodejs.org/api/addons.html) build/Release -# Publican runner -publican +# Bikeshed clone +bikeshed # Dependency directory # Commenting this out is preferred by some people, see diff --git a/.npmrc b/.npmrc new file mode 100644 index 0000000..cf905b0 --- /dev/null +++ b/.npmrc @@ -0,0 +1 @@ +prefix = ${HOME}/node_modules diff --git a/DOCKER.md b/DOCKER.md index 4d913a1..60df16e 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -26,9 +26,9 @@ If you changed project source, you’ll need to build another docker container. The following takes into account that you can run `docker` commands. ```bash -git checkout task-based -make build -make +make clone-bikeshed +make docker-build +make docker-run ``` Which extends into something similar to: @@ -37,6 +37,10 @@ Which extends into something similar to: docker run -it --rm -v /Users/renoirb/workspaces/webplatform/service-publican/repo/data:/srv/webplatform/specs/data -p 8002:7002 webspecs/publican bin/run.sh ``` +... notice that the `-v /full/path/host:/full/path/container` option requires FULL path in both sides of the column + +... notice that the `-v /full/path/host:/full/path/container` option requires FULL path in both sides of the column + Refer to the appropriate documentation if you want to [install](https://docs.docker.com/installation/#installation) and run this project from a Docker container. diff --git a/Dockerfile b/Dockerfile index fd2d41e..193bca3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,35 +8,50 @@ FROM nodesource/trusty:0.10.36 MAINTAINER Renoir Boulanger -RUN mkdir -p /srv/webplatform/specs/data/queue && \ - mkdir -p /srv/webplatform/specs/data/logs && \ - mkdir -p /srv/webplatform/specs/bikeshed - +# Dependencies: Bikeshed, PhantomJS, Bikshed’s lxml RUN apt-get update && \ apt-get install -yqq git python2.7 python-dev python-pip libxslt1-dev libxml2-dev zlib1g-dev && \ - pip install lxml - -RUN /usr/sbin/useradd --home-dir /srv/webplatform/specs --shell /bin/bash nonroot && \ - /usr/sbin/adduser nonroot sudo - + apt-get install -yqq libfontconfig1 libfreetype6 curl && \ + apt-get autoremove -yqq --purge && \ + pip install --upgrade lxml + +# Copy everything we have locally into the container +# REMINDER: Make sure you run `make clone-bikeshed`, we prefer to keep a copy locally outside +# of the data volume. Otherwise it would make problems saying that bikeshed clone is not in the +# same filesystem. COPY . /srv/webplatform/specs/ -RUN chown -R nonroot:nonroot /srv/webplatform/specs - -USER nonroot - -RUN git clone -b webspecs https://github.com/webspecs/bikeshed.git /srv/webplatform/specs/bikeshed && \ - pip install --user --editable /srv/webplatform/specs/bikeshed - +# Make sure we have a "non root" user and +# delete any local workbench data/ directory +RUN /usr/sbin/useradd --system -G sudo --home-dir /srv/webplatform/specs --shell /bin/bash app-user && \ + sed -i '/^%sudo/d' /etc/sudoers && \ + echo '%sudo ALL=NOPASSWD: ALL' >> /etc/sudoers && \ + mv /srv/webplatform/specs/bikeshed /opt && \ + rm -rf data && \ + mkdir -p data/temp && \ + rm -rf Dockerfile Makefile .git .gitignore DOCKER.md && \ + chown -R app-user:www-data /srv/webplatform/specs && \ + chown -R app-user:app-user /opt/bikeshed + +# Switch from root to app-user user +# It **HAS to be** the SAME uid/gid as the owner on the host from which we’ll use as volume +USER app-user + +# Where the session will start from WORKDIR /srv/webplatform/specs -ENV PATH /srv/webplatform/specs/bin:/srv/webplatform/specs/.local/bin:$PATH +# Environment variables +ENV PATH /srv/webplatform/specs/node_modules/.bin:/srv/webplatform/specs/bin:/srv/webplatform/specs/.local/bin:$PATH ENV HOME /srv/webplatform/specs +ENV TMPDIR /srv/webplatform/specs/data/temp +ENV NODE_ENV production ENV GIT_DISCOVERY_ACROSS_FILESYSTEM true -RUN chmod +x bin/run.sh && \ - npm install +# Run what `make deps` would do +RUN pip install --upgrade --user --editable /opt/bikeshed && \ + mkdir -p node_modules && npm install +# Declare which port we expect to expose EXPOSE 7002 # Allow cli entry for debug, but make sure docker-compose.yml uses "command: bin/run.sh" diff --git a/Makefile b/Makefile index 7332305..02648d6 100644 --- a/Makefile +++ b/Makefile @@ -1,25 +1,24 @@ SHELL := bash -default: run +default: deps -deps: - mkdir -p bikeshed +clone-bikeshed: datadirs git clone -b webspecs https://github.com/webspecs/bikeshed.git bikeshed + +datadirs: + mkdir -p $(CURDIR)/data/{queue,gits,logs,temp} + +# Make sure Dockerfile has equivalent at note about `make deps` +deps: pip install --user --editable bikeshed + npm install docker-build: docker build -t webspecs/publican . docker-bash: - docker run -it --rm -v "$(CURDIR)/data":/srv/webplatform/specs/data -p 7002 webspecs/publican + docker run -it --rm -v "$(CURDIR)/data":/srv/webplatform/specs/data -p 7002:7002 webspecs/publican:latest docker-run: - mkdir -p $(CURDIR)/data/{logs,publish,queue} - docker run -it --rm -v "$(CURDIR)/data":/srv/webplatform/specs/data -p 7002 webspecs/publican bin/run.sh - docker ps | grep publican - -tail: - tail -f data/logs/all.log - -run: docker-run tail + docker run -it --rm -v "$(CURDIR)/data":/srv/webplatform/specs/data -p 7002:7002 webspecs/publican:latest bin/run.sh diff --git a/data/config.json.dist b/data/config.json.dist index ed7c3e5..d557631 100644 --- a/data/config.json.dist +++ b/data/config.json.dist @@ -1,5 +1,5 @@ { - "rsyncRemote": "ubuntu@specs.production.wpdn" + "bikeshed": "/opt/bikeshed/bikeshed.py" , "rsyncPath": "/srv/webplatform/specs/" , "python": "python2" , "logFile": "logs/all.log" diff --git a/lib/lock.js b/lib/lock.js index e6e6f16..b543582 100644 --- a/lib/lock.js +++ b/lib/lock.js @@ -3,19 +3,19 @@ var lock = require("lock")() , timeout = 1000 * 60 * 10 // 10 minutes ; -exports.createLock = function (man) { - return function (key, action, done) { +exports.createLock = function createLockHandler (man) { + return function createLockReturner (key, action, done) { man.log.info("Locking " + key); - lock(key, function (release) { + lock(key, function createLockLocker (release) { man.log.log("silly", "Processing locked key " + key); var tid = setTimeout(function () { man.log.error("Timeout on lock expired, releasing " + key + " with risk of conflict."); release(done)(); }, timeout); - action(function (err) { + action(function createLockLockerAction (err) { man.log.info("Releasing lock " + key); clearTimeout(tid); - release(function () { done(err); })(); + release(function createLockRelease () { done(err); })(); }); }); }; diff --git a/lib/manager.js b/lib/manager.js index fd45b17..b9b59ae 100644 --- a/lib/manager.js +++ b/lib/manager.js @@ -25,16 +25,16 @@ function Manager (opt) { } Manager.prototype = { // run this with extreme caution - initSetup: function (cb) { + initSetup: function initSetupHandler (cb) { this.log.info("Initialising setup"); - this.runTask(require("./tasks/init-all"), this.getContext(), function (err) { + this.runTask(require("./tasks/init-all"), this.getContext(), function runTaskErrorHandler (err) { if (err) return this.log.error(err); this.log.info("OK!"); if (cb) cb(); }.bind(this)); } -, updateContextForRepo: function (ctx, opt) { +, updateContextForRepo: function updateContextForRepoHandler (ctx, opt) { if (opt.repository) { // git details ctx.gitRepoURL = "https://github.com/" + opt.repository + ".git"; @@ -48,15 +48,15 @@ Manager.prototype = { } } -, getContext: function (opt) { +, getContext: function getContextHandler (opt) { opt = opt || {}; var ctx = { // common facilities log: this.log , lock: lock.createLock(this) - , rfs: function (file) { return fs.readFileSync(file, "utf8"); } - , wfs: function (file, content) { fs.writeFileSync(file, content, { encoding: "utf8" }); } - , buildSpecDir: function (repo, branch) { + , rfs: function readFileSyncHandler (file) { return fs.readFileSync(file, "utf8"); } + , wfs: function writeFileSyncHandler (file, content) { fs.writeFileSync(file, content, { encoding: "utf8" }); } + , buildSpecDir: function buildSpecDirHandler (repo, branch) { var parts = repo.split("/", 2); return [parts[1], parts[0], branch].join("/"); } @@ -65,7 +65,7 @@ Manager.prototype = { , publishDir: jn(dataDir, "publish") , theIndexPath: jn(dataDir, "publish/index.html") // bikeshed details (conf has python) - , bikeshed: jn(dataDir, "bikeshed/bikeshed.py") + , bikeshed: "bikeshed" } ; this.updateContextForRepo(ctx, opt); @@ -74,15 +74,15 @@ Manager.prototype = { return ctx; } -, runTask: function (tasks, ctx, cb) { +, runTask: function runTaskHandler (tasks, ctx, cb) { if (!ctx) ctx = this.getContext(); if (!util.isArray(tasks)) tasks = [tasks]; tasks = tasks.concat(); - var done = function (err) { + var done = function runTaskDoneHandler (err) { if (err) return cb(err); iterate(); } - , iterate = function () { + , iterate = function runTaskIterateHandler() { if (!tasks.length) return cb(); var task = tasks.shift(); // console.log("####### tasks: " + tasks.length); @@ -98,7 +98,7 @@ Manager.prototype = { // this is the entry point for the manager, called by the server // it looks at repoDesc (which has a repository and branch) and based on that it will // pick a task to run, having prepared the correct context for it -, runAppropriateTask: function (repoDesc, cb) { // replaces processRepository +, runAppropriateTask: function funAppropriateTaskHandler (repoDesc, cb) { // replaces processRepository var repo = repoDesc.repository , ctx = this.getContext(repoDesc) ; diff --git a/lib/queue.js b/lib/queue.js index 7051ab3..45ab56d 100644 --- a/lib/queue.js +++ b/lib/queue.js @@ -5,24 +5,24 @@ var fs = require("fs") , queueDir = jn(dataDir, "queue") ; -exports.enqueue = function (repo, branch, cb) { +exports.enqueue = function enqueueHandler (repo, branch, cb) { var stamp = Date.now() + "-" + process.hrtime()[1]; fs.writeFile( jn(queueDir, stamp + ".json") , JSON.stringify({ repository: repo, branch: branch }, null, 4) , { encoding: "utf8" } - , function (err) { + , function enqueueCallbackHandler (err) { if (err) throw err; cb(null, stamp); } ); }; -exports.next = function (cb) { - fs.readdir(queueDir, function (err, queue) { +exports.next = function enqueueNextHandler (cb) { + fs.readdir(queueDir, function enqueueNextReadDirHandler (err, queue) { if (err) throw err; queue = queue - .sort(function (a, b) { // we have to do this to guarantee numeric comparison + .sort(function enqueueNextReadDirSortHandler (a, b) { // we have to do this to guarantee numeric comparison if (a < b) return -1; if (a > b) return 1; return 0; @@ -31,10 +31,10 @@ exports.next = function (cb) { var top = queue[0] , file = jn(queueDir, top) ; - fs.readFile(file, { encoding: "utf8" }, function (err, content) { + fs.readFile(file, { encoding: "utf8" }, function enqueueNextReadFileHandler (err, content) { if (err) throw err; var content = JSON.parse(content); - fs.unlink(file, function (err) { + fs.unlink(file, function enqueueNextReadFileUnlinkHandler (err) { if (err) throw err; cb(null, content); }); diff --git a/lib/tasks/extract-the-index.js b/lib/tasks/extract-the-index.js index 139cb0c..f05dd55 100644 --- a/lib/tasks/extract-the-index.js +++ b/lib/tasks/extract-the-index.js @@ -8,13 +8,13 @@ var whacko = require("whacko") // sets // theIndexRepositories: an array of repositories extracted from the-index // it has: branch, repository, baseFileName -module.exports = function (ctx, cb) { +module.exports = function extractTheIndex (ctx, cb) { ctx.theIndexRepositories = []; if (!fs.existsSync(ctx.theIndexPath)) return cb(); var $ = whacko.load(ctx.rfs(ctx.theIndexPath)); ctx.log.info("Extracting data from the-index"); $("script[type='application/webspec+json']") - .each(function () { + .each(function extractTheIndexSeekLoop () { try { var data = JSON.parse($(this).text()); for (var k in data) { diff --git a/lib/tasks/generate-bikeshed.js b/lib/tasks/generate-bikeshed.js index 885e7e7..f8bda43 100644 --- a/lib/tasks/generate-bikeshed.js +++ b/lib/tasks/generate-bikeshed.js @@ -8,19 +8,19 @@ var jn = require("path").join // bikeshed: the path to bikeshed // specDir: the path to the directory in which the spec lives // baseFileName: the base file name, without extension, to use (defaults to index) -module.exports = function (ctx, cb) { +module.exports = function generateBikeshed (ctx, cb) { var fn = ctx.baseFileName; if (fn.indexOf("/") === 0 || fn.indexOf("..") > -1) return cb("Invalid file name: " + fn); ctx.lock( "bikeshed " + ctx.specDir - , function (release) { + , function generateBikeshedLockHandler (release) { var cmd = ctx.python + " " + ctx.bikeshed + " -f spec " + jn(ctx.specDir, fn + ".bs") + " " + jn(ctx.specDir, "index.html") ; ctx.log.info(cmd); - exec(cmd, { cwd: ctx.specDir }, function (err, stdout, stderr) { + exec(cmd, { cwd: ctx.specDir }, function generateBikeshedExecHandler (err, stdout, stderr) { if (err) { ctx.log.error(stderr); ctx.log.error(stdout); diff --git a/lib/tasks/generate-respec.js b/lib/tasks/generate-respec.js index def3324..8f7d7fb 100644 --- a/lib/tasks/generate-respec.js +++ b/lib/tasks/generate-respec.js @@ -3,7 +3,7 @@ var jn = require("path").join , phantom = require("phantomjs") , execFile = require("child_process").execFile , querystring = require("querystring") -, num2 = function (num) { +, num2 = function generateRespecNum (num) { var str = num + ""; if (str.length >= 2) return str; return "0" + str; @@ -14,14 +14,14 @@ var jn = require("path").join // specDir: the path to the directory in which the spec lives // baseFileName: the base file name, without extension, to use (defaults to index) // repository: the repository in user/repo form -module.exports = function (ctx, cb) { +module.exports = function generateRespec (ctx, cb) { var fn = ctx.baseFileName , respec2html = jn(ctx.dataDir, "../node_modules/respec/tools/respec2html.js") ; if (fn.indexOf("/") === 0 || fn.indexOf("..") > -1) return cb("Invalid file name: " + fn); ctx.lock( "respec " + ctx.specDir - , function (release) { + , function generateRespecLockHandler (release) { var d = new Date() , enforce = { specStatus: "webspec" @@ -37,10 +37,10 @@ module.exports = function (ctx, cb) { ] ; ctx.log.info(phantom.path + " " + params.join(" ")); - + var timedOut = false , tickingBomb = setTimeout( - function () { + function generateRespecReleaseHandler () { timedOut = true; release(new Error("ReSpec timed out.")); } @@ -50,7 +50,7 @@ module.exports = function (ctx, cb) { execFile( phantom.path , params - , function (err, stdout, stderr) { + , function generateRespecExecHandler (err, stdout, stderr) { if (timedOut) return; clearTimeout(tickingBomb); if (err) { diff --git a/lib/tasks/generate-the-index.js b/lib/tasks/generate-the-index.js index 2e87bff..eb04516 100644 --- a/lib/tasks/generate-the-index.js +++ b/lib/tasks/generate-the-index.js @@ -6,20 +6,20 @@ var whacko = require("whacko") // expects // theIndexPath: the path to the-index file // publishDir: the directory to publish to -module.exports = function (ctx, cb) { +module.exports = function generateTheIndex (ctx, cb) { var $ = whacko.load(ctx.rfs(ctx.theIndexPath)); // make the tables ctx.log.info("Generating specification linking tables in the-index"); $("script[type='application/webspec+json']") - .each(function () { + .each(function generateTheIndexSeekHandler () { var $script = $(this) , ws = JSON.parse($script.text()) , $dl = $("
") - , simpleDD = function (desc, $dl) { + , simpleDD = function generateTheIndexDdHonk (desc, $dl) { $dl.append($("
").html(desc || "n/a")); } - , linkDT = function (repo, $dl) { + , linkDT = function generateTheIndexDtHonk (repo, $dl) { var parts = repo.split("/", 2) , user = parts[0] , $dt = $("
"); @@ -41,12 +41,12 @@ module.exports = function (ctx, cb) { $script.after($dl); }) ; - + // build the ToC var toc = []; ctx.log.info("Building ToC for the-index"); $("section") - .each(function () { + .each(function generateTheIndexSectionHonk () { var $section = $(this) , topLevel = !!$section.find("h2").length , $h = $section.find(topLevel ? "h2" : "h3").first() @@ -79,7 +79,7 @@ module.exports = function (ctx, cb) { if (!$h.hasClass("no-ref")) $h.append($("").attr("href", "#" + id)); }) ; - var tocline = function (details, $parent) { + var tocline = function generateTheIndexTocLineHonk (details, $parent) { var $li = $("
    • ") , $a = $li.find("a") ; diff --git a/lib/tasks/git-clone-or-fetch.js b/lib/tasks/git-clone-or-fetch.js index 266e224..17a8e21 100644 --- a/lib/tasks/git-clone-or-fetch.js +++ b/lib/tasks/git-clone-or-fetch.js @@ -6,8 +6,8 @@ var fs = require("fs-extra") // expects: // gitDir: the directory in which the repo is stored -module.exports = function (ctx, cb) { - fs.exists(ctx.gitDir, function (exists) { +module.exports = function gitCloneOrFetchExports (ctx, cb) { + fs.exists(ctx.gitDir, function gitCloneOrFetchFileExistsHandler (exists) { this.runTask(exists ? fetch : clone, ctx, cb); }.bind(this)); }; diff --git a/lib/tasks/git-clone.js b/lib/tasks/git-clone.js index f6946f2..89a3b29 100644 --- a/lib/tasks/git-clone.js +++ b/lib/tasks/git-clone.js @@ -6,11 +6,11 @@ var fs = require("fs-extra") // expects: // gitDir: the directory in which the repo is stored // gitRepoURL: the repository -module.exports = function (ctx, cb) { +module.exports = function gitClone (ctx, cb) { ctx.lock( "clone " + ctx.gitRepoURL - , function (release) { - fs.mkdirp(ctx.gitDir, function (err) { + , function gitCloneLockHandler (release) { + fs.mkdirp(ctx.gitDir, function gitCloneMkdirHandler (err) { if (err) return release(err); exec("git clone --mirror " + ctx.gitRepoURL + " " + ctx.gitDir, release); }); diff --git a/lib/tasks/git-fetch.js b/lib/tasks/git-fetch.js index e5b9d6a..f5f870b 100644 --- a/lib/tasks/git-fetch.js +++ b/lib/tasks/git-fetch.js @@ -3,10 +3,10 @@ var exec = require("child_process").exec; // expects: // gitDir: the directory in which the repo is stored -module.exports = function (ctx, cb) { +module.exports = function gitFetch (ctx, cb) { ctx.lock( "fetch " + ctx.gitDir - , function (release) { + , function gitFetchHandler (release) { ctx.log.info("git fetch in " + ctx.gitDir); exec("git fetch", { cwd: ctx.gitDir }, release); } diff --git a/lib/tasks/git-publish.js b/lib/tasks/git-publish.js index 79a506c..a66f866 100644 --- a/lib/tasks/git-publish.js +++ b/lib/tasks/git-publish.js @@ -12,7 +12,7 @@ var fs = require("fs-extra") // copyFile: if defined, the file name to copy over instead of the whole directory module.exports = [ // make a temp directory - function (ctx, cb) { + function gitPublish (ctx, cb) { tmp.dir({ mode: 0755 }, function (err, tmpDir) { if (err) return cb(err); ctx.tmpDir = tmpDir; @@ -20,16 +20,16 @@ module.exports = [ }); } // extract the content - , function (ctx, cb) { + , function gitPublishArchiveHandler (ctx, cb) { var cmd = "git archive " + ctx.branch + " | tar -x -C " + ctx.tmpDir; ctx.log.info("Publish: " + cmd + ", in " + ctx.gitDir); exec(cmd, { cwd: ctx.gitDir }, cb); } // move over the content - , function (ctx, cb) { + , function gitPublishArchiveMover (ctx, cb) { ctx.lock( "move over content to " + ctx.specDir - , function (release) { + , function gitPublishFileLockHandler (release) { var tmp = ctx.tmpDir; delete ctx.tmpDir; if (ctx.copyFile) { @@ -41,7 +41,7 @@ module.exports = [ } , cb ); - + } ] ; diff --git a/lib/tasks/init-all.js b/lib/tasks/init-all.js index 3854aa8..5ca1107 100644 --- a/lib/tasks/init-all.js +++ b/lib/tasks/init-all.js @@ -5,7 +5,7 @@ var fs = require("fs-extra") ; function makeStaticTask (which) { - return function (ctx, cb) { + return function initAllMakeStatic (ctx, cb) { ctx = this.getContext({ repository: "webspecs/" + which, branch: "master" }); ctx.subDirOnly = which; ctx.specDir = jn(ctx.publishDir, ctx.subDirOnly); @@ -14,10 +14,10 @@ function makeStaticTask (which) { } module.exports = [ - function (ctx, cb) { + function initAll (ctx, cb) { "gits publish bikeshed queue logs" .split(" ") - .forEach(function (dir) { + .forEach(function initAllDirMaker (dir) { dir = jn(ctx.dataDir, dir); ctx.log.info("Ensuring the existence of " + dir); if (!fs.existsSync(dir)) fs.mkdirSync(dir); @@ -30,7 +30,7 @@ module.exports = [ , require("./init-bikeshed") , require("./init-the-index") , require("./extract-the-index") - , function (ctx, cb) { + , function initAllCallbackHandler (ctx, cb) { ctx.specsToProcess = ctx.theIndexRepositories; cb(); } diff --git a/lib/tasks/init-bikeshed.js b/lib/tasks/init-bikeshed.js index d931ba9..76c7cc4 100644 --- a/lib/tasks/init-bikeshed.js +++ b/lib/tasks/init-bikeshed.js @@ -2,9 +2,9 @@ var jn = require("path").join; module.exports = [ - function (ctx, cb) { + function initBikeshed (ctx, cb) { this.updateContextForRepo(ctx, { repository: "webspecs/bikeshed", branch: "webspecs" }); - ctx.specDir = jn(ctx.dataDir, "bikeshed"); + ctx.specDir = "bikeshed"; cb(); } , require("./git-clone-or-fetch") diff --git a/lib/tasks/init-the-index.js b/lib/tasks/init-the-index.js index b4a9398..10c8bde 100644 --- a/lib/tasks/init-the-index.js +++ b/lib/tasks/init-the-index.js @@ -18,6 +18,5 @@ module.exports = [ ctx.subDirOnly = "index.html"; cb(); } - , require("./purge-all") ] ; diff --git a/lib/tasks/process-spec.js b/lib/tasks/process-spec.js index 9a99f94..b4edc23 100644 --- a/lib/tasks/process-spec.js +++ b/lib/tasks/process-spec.js @@ -28,6 +28,5 @@ module.exports = [ return this.runTask(require("./generate-bikeshed"), ctx, cb); cb(new Error("Failed to find a specification file in " + ctx.specDir)); } - , require("./purge-all") ] ; diff --git a/lib/tasks/process-specs.js b/lib/tasks/process-specs.js index fadc469..342f6c3 100644 --- a/lib/tasks/process-specs.js +++ b/lib/tasks/process-specs.js @@ -3,16 +3,16 @@ var processSpec = require("./process-spec"); // expects // specsToProcess: an array of specs to process much like `theIndexRepositories` -module.exports = function (ctx, cb) { +module.exports = function processSpecs (ctx, cb) { var specs = ctx.specsToProcess.concat(); ctx.log.info("Processing " + specs.length + " specifications."); - var iterate = function () { + var iterate = function processSpecsIterate () { if (!specs.length) return cb(); var it = specs.shift() , ctx = this.getContext(it) ; ctx.subDirOnly = ctx.specSubDir; - this.runTask(processSpec, ctx, function (err) { + this.runTask(processSpec, ctx, function processSpecsRunTask (err) { if (err) ctx.log.error(err); iterate(); }); diff --git a/lib/tasks/process-static.js b/lib/tasks/process-static.js index 8daeaea..3752d2f 100644 --- a/lib/tasks/process-static.js +++ b/lib/tasks/process-static.js @@ -2,6 +2,5 @@ module.exports = [ require("./git-clone-or-fetch") , require("./git-publish") - , require("./purge-all") ] ; diff --git a/lib/tasks/process-the-index.js b/lib/tasks/process-the-index.js index 3765f03..7f47a19 100644 --- a/lib/tasks/process-the-index.js +++ b/lib/tasks/process-the-index.js @@ -1,19 +1,19 @@ module.exports = [ require("./extract-the-index") // first time, get the old values - , function (ctx, cb) { + , function processTheIndexFirst (ctx, cb) { ctx.oldIndexRepositories = ctx.theIndexRepositories; ctx.theIndexRepositories = null; cb(); } , require("./init-the-index") , require("./extract-the-index") // get the updated list - , function (ctx, cb) { + , function processTheIndexSecond (ctx, cb) { ctx.log.info("Finding new specifications to process"); var newRepos = []; ctx.theIndexRepositories - .forEach(function (it) { - var isOld = ctx.oldIndexRepositories.some(function (old) { + .forEach(function processTheIndexRepositoryLooper (it) { + var isOld = ctx.oldIndexRepositories.some(function processTheIndexOldIsser (old) { return old.repository === it.repository && old.branch === it.branch && old.baseFileName === it.baseFileName; From d95d1e510d3021fbf7abb86e73bac4d6e2a7b7d3 Mon Sep 17 00:00:00 2001 From: Renoir Boulanger Date: Mon, 11 May 2015 17:59:39 -0400 Subject: [PATCH 11/18] Added screenshots from PR webspecs/publican#21 --- DOCKER.md | 40 ++++++++++++++++++++++++++++++++-------- Makefile | 5 +++-- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/DOCKER.md b/DOCKER.md index 60df16e..3be64af 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -9,8 +9,8 @@ The set of tools used to automate publishing in the WebSpecs project If you have [Docker installed](https://docs.docker.com/installation/) and running, you can run it in a few commands. ```bash -docker pull webspecs/publican -docker run -it --rm -p 7002:7002 webspecs/publican bin/run.sh +docker pull webspecs/publican:latest +docker run -it --rm -p 7002:7002 -v "$(pwd)/data":/srv/webplatform/specs/data -v "$(pwd)/spec-data":/opt/bikeshed/bikeshed/spec-data webspecs/publican:latest bin/run.sh ``` If you want to run with your own customizations, you’d have to do also have [**Docker Compose** installed](https://docs.docker.com/compose/install/) and do the following. @@ -34,24 +34,48 @@ make docker-run Which extends into something similar to: ``` -docker run -it --rm -v /Users/renoirb/workspaces/webplatform/service-publican/repo/data:/srv/webplatform/specs/data -p 8002:7002 webspecs/publican bin/run.sh +docker run -it --rm -p 7002:7002 \ + -v /Users/renoirb/workspaces/webplatform/service-publican/repo/data:/srv/webplatform/specs/data \ + -v /Users/renoirb/workspaces/webplatform/service-publican/repo/spec-data:/opt/bikeshed/bikeshed/spec-data \ + webspecs/publican:latest bin/run.sh ``` ... notice that the `-v /full/path/host:/full/path/container` option requires FULL path in both sides of the column -... notice that the `-v /full/path/host:/full/path/container` option requires FULL path in both sides of the column Refer to the appropriate documentation if you want to [install](https://docs.docker.com/installation/#installation) and run this project from a Docker container. +## Run an instance locally + +Run a container, and send a hook call + + make docker-run + +On another terminal tab, send a hook; + + curl -H "Content-Type: application/json" -XPOST localhost:7002/hook -d '{"base_ref":"master","repository":{"name":"assets","owner":{"name":"webspecs"}}}' + + +you should see something like; + + {"ok":true,"details":"Queued 1431380611165-28770219 for processing."} + +And in the container, you’d see; + +![publican-run-hook](https://cloud.githubusercontent.com/assets/296940/7575805/f18cb1d2-f805-11e4-8ec3-dba68dae7785.png) + + ## Enter the container shell -This is useful to understand what’s happening +You can initialize an empty workspace using `publican.js init`, it’ll write in `data/` that should already be mounted through the `Makefile`. -```bash -make bash -``` +Its also useful to understand what’s happening behind the scenes. + + make docker-bash + +![publican-init](https://cloud.githubusercontent.com/assets/296940/7575777/b5a76176-f805-11e4-99e7-3a7c58dd304a.png) diff --git a/Makefile b/Makefile index 02648d6..4d9cb1d 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ clone-bikeshed: datadirs datadirs: mkdir -p $(CURDIR)/data/{queue,gits,logs,temp} + mkdir -p $(CURDIR)/spec-data # Make sure Dockerfile has equivalent at note about `make deps` deps: @@ -17,8 +18,8 @@ docker-build: docker build -t webspecs/publican . docker-bash: - docker run -it --rm -v "$(CURDIR)/data":/srv/webplatform/specs/data -p 7002:7002 webspecs/publican:latest + docker run -it --rm -v "$(CURDIR)/data":/srv/webplatform/specs/data -v "$(CURDIR)/spec-data":/opt/bikeshed/bikeshed/spec-data -p 7002:7002 webspecs/publican:latest docker-run: - docker run -it --rm -v "$(CURDIR)/data":/srv/webplatform/specs/data -p 7002:7002 webspecs/publican:latest bin/run.sh + docker run -it --rm -v "$(CURDIR)/data":/srv/webplatform/specs/data -v "$(CURDIR)/spec-data":/opt/bikeshed/bikeshed/spec-data -p 7002:7002 webspecs/publican:latest bin/run.sh From 704b6d417049b36f23ff8c7773dd87cdbabe03a9 Mon Sep 17 00:00:00 2001 From: Renoir Boulanger Date: Mon, 11 May 2015 18:13:05 -0400 Subject: [PATCH 12/18] More notes in DOCKER.md --- DOCKER.md | 65 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 25 deletions(-) diff --git a/DOCKER.md b/DOCKER.md index 3be64af..e0a2389 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -1,13 +1,9 @@ -# Publican - -The set of tools used to automate publishing in the WebSpecs project - - - -## Run in Docker +# Run Publican in Docker If you have [Docker installed](https://docs.docker.com/installation/) and running, you can run it in a few commands. +It can summarize as; + ```bash docker pull webspecs/publican:latest docker run -it --rm -p 7002:7002 -v "$(pwd)/data":/srv/webplatform/specs/data -v "$(pwd)/spec-data":/opt/bikeshed/bikeshed/spec-data webspecs/publican:latest bin/run.sh @@ -18,35 +14,49 @@ If you want to run with your own customizations, you’d have to do also have [* Follow the instructions in **Run using _Docker Compose_** in [the registry description instructions](https://registry.hub.docker.com/u/webspecs/publican/) +## Other possible usage -## Build Docker container +### Build Docker container -If you changed project source, you’ll need to build another docker container. +When we call `docker pull webspecs/publican:latest`, its either calling a last successful Docker build command that has the `-t webspecs/publican`. -The following takes into account that you can run `docker` commands. +In order to deploy, we need to have an image to run, that’s why you have to build one. -```bash -make clone-bikeshed -make docker-build -make docker-run -``` +To build a **webspecs/publican** container, you need to use the publican repository. -Which extends into something similar to: +Once you have a clone of publican, get the dependencies; -``` -docker run -it --rm -p 7002:7002 \ + make clone-bikeshed + +Which will clone bikeshed and create required directories for you. + +Build the container + + make docker-build + +Run the container + + make docker-run + +If you review what’s in the [Makefile][./Makefile], you’ll see that it basicall expands to something like this; + + docker run -it --rm -p 7002:7002 \ -v /Users/renoirb/workspaces/webplatform/service-publican/repo/data:/srv/webplatform/specs/data \ -v /Users/renoirb/workspaces/webplatform/service-publican/repo/spec-data:/opt/bikeshed/bikeshed/spec-data \ webspecs/publican:latest bin/run.sh -``` -... notice that the `-v /full/path/host:/full/path/container` option requires FULL path in both sides of the column +Notice that the `-v /full/path/host:/full/path/container` option requires FULL path in both sides of the column + +The last part of the `docker run` is what’s going to run, in this case `bin/run.sh`. +Which means you could get into a running container shell if you need to. +That’s what the `make docker-bash` do. +j Refer to the appropriate documentation if you want to [install](https://docs.docker.com/installation/#installation) and run this project from a Docker container. -## Run an instance locally +### Run an instance locally Run a container, and send a hook call @@ -67,7 +77,7 @@ And in the container, you’d see; -## Enter the container shell +### Enter the container shell You can initialize an empty workspace using `publican.js init`, it’ll write in `data/` that should already be mounted through the `Makefile`. @@ -79,14 +89,14 @@ Its also useful to understand what’s happening behind the scenes. -## Push Docker container to Docker Hub registry +### Push Docker container to Docker Hub registry The Docker registry is basically a storage service of docker container states from which we can pull and run from it. It saves us to rebuild the container on every server that would run the container. Docker uses a semantic similar to git, a container can be commited and pushed to a repository. That’s what we’ll do here. -Rever to [Docker command line "commit" documentation](http://docs.docker.com/reference/commandline/cli/#commit-a-container) for further details. +Refer to [Docker command line "commit" documentation](http://docs.docker.com/reference/commandline/cli/#commit-a-container) for further details. This project Docker registry entry is at [registry.hub.docker.com/u/**webspecs/publican**](https://registry.hub.docker.com/u/webspecs/publican/). @@ -112,7 +122,12 @@ Done. -## Run from Docker composer +### Run from Docker composer TODO, see notes in [**webspecs/publican**](https://registry.hub.docker.com/u/webspecs/publican/) Docker hub description. + +## Reference + +Refer to the appropriate documentation if you want to [install](https://docs.docker.com/installation/#installation) +and run this project from a Docker container locally. From 649d47be13c5ab928ab84b80027e08a5345d5b1c Mon Sep 17 00:00:00 2001 From: Renoir Boulanger Date: Mon, 11 May 2015 18:14:45 -0400 Subject: [PATCH 13/18] Typo --- DOCKER.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/DOCKER.md b/DOCKER.md index e0a2389..436657f 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -38,7 +38,7 @@ Run the container make docker-run -If you review what’s in the [Makefile][./Makefile], you’ll see that it basicall expands to something like this; +If you review what’s in the [Makefile](./Makefile), you’ll see that it basicall expands to something like this; docker run -it --rm -p 7002:7002 \ -v /Users/renoirb/workspaces/webplatform/service-publican/repo/data:/srv/webplatform/specs/data \ @@ -52,7 +52,6 @@ Which means you could get into a running container shell if you need to. That’s what the `make docker-bash` do. -j Refer to the appropriate documentation if you want to [install](https://docs.docker.com/installation/#installation) and run this project from a Docker container. From 48e52b4413eb63f9b60d7522e9fe8be89aaed49b Mon Sep 17 00:00:00 2001 From: Renoir Boulanger Date: Mon, 11 May 2015 18:17:33 -0400 Subject: [PATCH 14/18] Added context reference --- DOCKER.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/DOCKER.md b/DOCKER.md index 436657f..0b728c1 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -18,12 +18,15 @@ Follow the instructions in **Run using _Docker Compose_** in [the registry descr ### Build Docker container -When we call `docker pull webspecs/publican:latest`, its either calling a last successful Docker build command that has the `-t webspecs/publican`. +When we call `docker pull webspecs/publican:latest`, its either calling a last successful Docker build command that has the `-t webspecs/publican` +or its asking to get the latest version from the Docker hub. In order to deploy, we need to have an image to run, that’s why you have to build one. To build a **webspecs/publican** container, you need to use the publican repository. +Once a build is successful you can push it to [Push Docker container to Docker Hub registry](#Push Docker container to Docker Hub registry). + Once you have a clone of publican, get the dependencies; make clone-bikeshed From 2ec16a1d3f793487c703a927712212cf87d527af Mon Sep 17 00:00:00 2001 From: Renoir Boulanger Date: Mon, 11 May 2015 18:45:08 -0400 Subject: [PATCH 15/18] Ensure respec.js init also adds data/temp --- lib/tasks/init-all.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/init-all.js b/lib/tasks/init-all.js index 5ca1107..4101720 100644 --- a/lib/tasks/init-all.js +++ b/lib/tasks/init-all.js @@ -15,7 +15,7 @@ function makeStaticTask (which) { module.exports = [ function initAll (ctx, cb) { - "gits publish bikeshed queue logs" + "gits publish bikeshed queue logs temp" .split(" ") .forEach(function initAllDirMaker (dir) { dir = jn(ctx.dataDir, dir); From 735f9cb68fee06ad5c69f016464abcd8e3726c4c Mon Sep 17 00:00:00 2001 From: Renoir Boulanger Date: Tue, 12 May 2015 13:59:52 -0400 Subject: [PATCH 16/18] webplatform/ops#153 honk > renoirb/salt-basesystem --- DOCKER.md | 2 +- Dockerfile | 29 ++++++++++++++++------------- Makefile | 4 ++-- bin/run.sh | 2 +- data/config.json.dist | 8 +++----- docker-compose.yml.sample | 29 +++++++++++++++++++++++++++++ 6 files changed, 52 insertions(+), 22 deletions(-) create mode 100644 docker-compose.yml.sample diff --git a/DOCKER.md b/DOCKER.md index 0b728c1..dd1381b 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -44,7 +44,7 @@ Run the container If you review what’s in the [Makefile](./Makefile), you’ll see that it basicall expands to something like this; docker run -it --rm -p 7002:7002 \ - -v /Users/renoirb/workspaces/webplatform/service-publican/repo/data:/srv/webplatform/specs/data \ + -v /Users/renoirb/workspaces/webplatform/service-publican/repo/data:/srv/webapps/publican/data \ -v /Users/renoirb/workspaces/webplatform/service-publican/repo/spec-data:/opt/bikeshed/bikeshed/spec-data \ webspecs/publican:latest bin/run.sh diff --git a/Dockerfile b/Dockerfile index 193bca3..feacebb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,8 +8,10 @@ FROM nodesource/trusty:0.10.36 MAINTAINER Renoir Boulanger +ENV DEBIAN_FRONTEND=noninteractive + # Dependencies: Bikeshed, PhantomJS, Bikshed’s lxml -RUN apt-get update && \ +RUN apt-get update && apt-get -y upgrade && \ apt-get install -yqq git python2.7 python-dev python-pip libxslt1-dev libxml2-dev zlib1g-dev && \ apt-get install -yqq libfontconfig1 libfreetype6 curl && \ apt-get autoremove -yqq --purge && \ @@ -19,31 +21,32 @@ RUN apt-get update && \ # REMINDER: Make sure you run `make clone-bikeshed`, we prefer to keep a copy locally outside # of the data volume. Otherwise it would make problems saying that bikeshed clone is not in the # same filesystem. -COPY . /srv/webplatform/specs/ +COPY . /srv/webapps/publican/ # Make sure we have a "non root" user and # delete any local workbench data/ directory -RUN /usr/sbin/useradd --system -G sudo --home-dir /srv/webplatform/specs --shell /bin/bash app-user && \ +RUN /usr/sbin/groupadd --system --gid 990 webapps && \ + /usr/sbin/useradd --system --gid 990 --uid 990 -G sudo --home-dir /srv/webapps --shell /bin/bash webapps && \ sed -i '/^%sudo/d' /etc/sudoers && \ echo '%sudo ALL=NOPASSWD: ALL' >> /etc/sudoers && \ - mv /srv/webplatform/specs/bikeshed /opt && \ + mv /srv/webapps/publican/bikeshed /opt && \ rm -rf data && \ mkdir -p data/temp && \ rm -rf Dockerfile Makefile .git .gitignore DOCKER.md && \ - chown -R app-user:www-data /srv/webplatform/specs && \ - chown -R app-user:app-user /opt/bikeshed + chown -R webapps:webapps /srv/webapps/publican && \ + chown -R webapps:webapps /opt/bikeshed -# Switch from root to app-user user +# Switch from root to webapps system user # It **HAS to be** the SAME uid/gid as the owner on the host from which we’ll use as volume -USER app-user +USER webapps # Where the session will start from -WORKDIR /srv/webplatform/specs +WORKDIR /srv/webapps/publican # Environment variables -ENV PATH /srv/webplatform/specs/node_modules/.bin:/srv/webplatform/specs/bin:/srv/webplatform/specs/.local/bin:$PATH -ENV HOME /srv/webplatform/specs -ENV TMPDIR /srv/webplatform/specs/data/temp +ENV PATH /srv/webapps/publican/node_modules/.bin:/srv/webapps/publican/bin:/srv/webapps/publican/.local/bin:$PATH +ENV HOME /srv/webapps/publican +ENV TMPDIR /srv/webapps/publican/data/temp ENV NODE_ENV production ENV GIT_DISCOVERY_ACROSS_FILESYSTEM true @@ -58,7 +61,7 @@ EXPOSE 7002 ENTRYPOINT ["/bin/bash"] # Note leftover: Ideally, it should exclusively run -#ENTRYPOINT ["/bin/bash", "/srv/webplatform/specs/bin/run.sh"] +#ENTRYPOINT ["/bin/bash", "/srv/webapps/publican/bin/run.sh"] # Note leftover: What it ends up doing #CMD ["node_modules/forever/bin/forever", "--fifo", "logs", "0"] diff --git a/Makefile b/Makefile index 4d9cb1d..a30370e 100644 --- a/Makefile +++ b/Makefile @@ -18,8 +18,8 @@ docker-build: docker build -t webspecs/publican . docker-bash: - docker run -it --rm -v "$(CURDIR)/data":/srv/webplatform/specs/data -v "$(CURDIR)/spec-data":/opt/bikeshed/bikeshed/spec-data -p 7002:7002 webspecs/publican:latest + docker run -it --rm -v "$(CURDIR)/data":/srv/webapps/publican/data -v "$(CURDIR)/spec-data":/opt/bikeshed/bikeshed/spec-data -p 7002:7002 webspecs/publican:latest docker-run: - docker run -it --rm -v "$(CURDIR)/data":/srv/webplatform/specs/data -v "$(CURDIR)/spec-data":/opt/bikeshed/bikeshed/spec-data -p 7002:7002 webspecs/publican:latest bin/run.sh + docker run -it --rm -v "$(CURDIR)/data":/srv/webapps/publican/data -v "$(CURDIR)/spec-data":/opt/bikeshed/bikeshed/spec-data -p 7002:7002 webspecs/publican:latest bin/run.sh diff --git a/bin/run.sh b/bin/run.sh index db56782..ae6f52d 100644 --- a/bin/run.sh +++ b/bin/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -export RUNDIR="/srv/webplatform/specs" +export RUNDIR="/srv/webapps/publican" cd $RUNDIR diff --git a/data/config.json.dist b/data/config.json.dist index d557631..8afba2f 100644 --- a/data/config.json.dist +++ b/data/config.json.dist @@ -1,14 +1,12 @@ { "bikeshed": "/opt/bikeshed/bikeshed.py" -, "rsyncPath": "/srv/webplatform/specs/" +, "rsyncPath": "/srv/webapps/publican/" , "python": "python2" , "logFile": "logs/all.log" , "email": { - "to": "robin@berjon.com" + "to": "team-webplatform-systems@w3.org" , "from": "publican-noreply@webplatform.org" - , "host": "mail.messagingengine.com" - , "port": 465 - , "username": "robin@berjon.com" + , "host": "localhost" , "password": "XXXX not the real thing, will not work. Should be replaced by a login that can email from here." , "ssl": true , "tls": false diff --git a/docker-compose.yml.sample b/docker-compose.yml.sample new file mode 100644 index 0000000..7e05beb --- /dev/null +++ b/docker-compose.yml.sample @@ -0,0 +1,29 @@ +main: + image: webspecs/publican:latest + + # Default in this container is bash, using command below runs + # publican server for us while allowing us to enter the original + # container image in case of need to understand what’s available. + command: bin/run.sh + + # Make sure you have a data folder in the same + # directory as this file. Container expects the following + # files and folders. + # - data/{queue,gits,logs,temp} + # - data/config.json + # - spec-data/read-only + volumes: + - data/:/srv/webapps/publican/data + - spec-data/:/opt/bikeshed/bikeshed/spec-data + + restart: always + + ports: + - "7002:7002" + + dns: + - 10.10.10.41 + - 8.8.8.8 + +# vim: et ts=2 sw=2 ft=yaml: + From 8e68119c4a94220024643de104d326476cff504d Mon Sep 17 00:00:00 2001 From: Renoir Boulanger Date: Wed, 20 May 2015 19:35:40 -0400 Subject: [PATCH 17/18] webspecs/publican#22 Proposal --- lib/manager.js | 3 +++ lib/tasks/process-spec.js | 1 + lib/tasks/process-static.js | 1 + lib/tasks/purge-all.js | 31 ++++++++++++++++++++++--------- lib/tasks/purge.js | 27 +++++++++++++++++++++++++++ package.json | 2 +- 6 files changed, 55 insertions(+), 10 deletions(-) create mode 100644 lib/tasks/purge.js diff --git a/lib/manager.js b/lib/manager.js index b9b59ae..e60e929 100644 --- a/lib/manager.js +++ b/lib/manager.js @@ -11,6 +11,7 @@ var fs = require("fs") , processStatic = require("./tasks/process-static") , processBikeshed = require("./tasks/init-bikeshed") , processSpec = require("./tasks/process-spec") +, purge = require("./tasks/purge") ; // loads the configuration itself @@ -119,7 +120,9 @@ Manager.prototype = { else { ctx.subDirOnly = ctx.specSubDir; this.runTask(processSpec, ctx, cb); + this.runTask(purge, ctx, cb); } } + }; module.exports = Manager; diff --git a/lib/tasks/process-spec.js b/lib/tasks/process-spec.js index b4edc23..098fa89 100644 --- a/lib/tasks/process-spec.js +++ b/lib/tasks/process-spec.js @@ -28,5 +28,6 @@ module.exports = [ return this.runTask(require("./generate-bikeshed"), ctx, cb); cb(new Error("Failed to find a specification file in " + ctx.specDir)); } + , require("./purge") ] ; diff --git a/lib/tasks/process-static.js b/lib/tasks/process-static.js index 3752d2f..8daeaea 100644 --- a/lib/tasks/process-static.js +++ b/lib/tasks/process-static.js @@ -2,5 +2,6 @@ module.exports = [ require("./git-clone-or-fetch") , require("./git-publish") + , require("./purge-all") ] ; diff --git a/lib/tasks/purge-all.js b/lib/tasks/purge-all.js index 56cecd6..337e0ac 100644 --- a/lib/tasks/purge-all.js +++ b/lib/tasks/purge-all.js @@ -1,16 +1,29 @@ -var sua = require("superagent"); +var request = require("request"); // expects: // purgeAllURL: the URL to purge Fastly // purgeAllKey: the key for the Fastly service module.exports = function (ctx, cb) { - sua.post(ctx.purgeAllURL) - .set("Fastly-Key", ctx.purgeAllKey) - .set('Accept', 'application/json') - .end(function (err, res) { - if (!err && res.body && res.body.status === "ok") return cb(); - cb(new Error(res.body ? res.body.status : "No response body for purge")); - }) - ; + + var options = { + url: ctx.purgeAllURL + , method: 'POST' + , headers: { + 'User-Agent': 'WebPlatformPublican/1' + , 'Fastly-Key': ctx.purgeAllKey + } + }; + + return request(options, function requestCallback(error, response, body) { + var info = JSON.parse(body); + if (!error && response.statusCode == 200) { + console.log('Purge worked', info); + return cb(); + } else { + cb( new Error('Error purging, message ', info ) ); + } + }); + }; + diff --git a/lib/tasks/purge.js b/lib/tasks/purge.js new file mode 100644 index 0000000..e679d61 --- /dev/null +++ b/lib/tasks/purge.js @@ -0,0 +1,27 @@ + +var request = require("request"); + +module.exports = function (ctx, cb) { + + var parts = ctx.repository.split("/", 2); + var options = { + url: 'https://specs.webplatform.org/' + , method: 'PURGE' + , headers: { + 'User-Agent': 'WebPlatformPublican/1' + } + }; + + options.url += parts[1] + '/' + parts[0] + '/' + ctx.branch; + + return request(options, function purgeRequestHandler (error, response, body) { + var info = JSON.parse(body); + if (!error && response.statusCode == 200) { + console.log('Purge worked on ' + options.url, info); + return cb(); + } else { + return cb(new Error('Error purging ' + options.url, info)); + } + }); + +}; diff --git a/package.json b/package.json index 2ce4e25..fea7d27 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "phantomjs": "1.9.15", "querystring": "^0.2.0", "respec": "3.2.40", - "superagent": "1.1.0", + "request": "2.42.0", "tmp": "0.0.25", "whacko": "0.17.4", "winston": "0.9.0", From 182bfad94b03c6f348391341ad875d697b0776f1 Mon Sep 17 00:00:00 2001 From: Renoir Boulanger Date: Tue, 26 May 2015 18:31:45 -0400 Subject: [PATCH 18/18] Support to not send purge if purgeAllKey is false --- lib/tasks/purge-all.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/tasks/purge-all.js b/lib/tasks/purge-all.js index 337e0ac..27a71c1 100644 --- a/lib/tasks/purge-all.js +++ b/lib/tasks/purge-all.js @@ -6,6 +6,10 @@ var request = require("request"); // purgeAllKey: the key for the Fastly service module.exports = function (ctx, cb) { + if ( ctx.purgeAllKey === false ) { + return cb(); + } + var options = { url: ctx.purgeAllURL , method: 'POST' @@ -21,7 +25,9 @@ module.exports = function (ctx, cb) { console.log('Purge worked', info); return cb(); } else { - cb( new Error('Error purging, message ', info ) ); + var msg = info.msg||'Request was refused by Fastly and the we could not find error message in response'; + console.error('Error purging, its possible the key is invalid, message: ', msg ); + cb( new Error('Error purging ' + msg ) ); } });