Skip to content

Commit 390edf4

Browse files
saraswatpuneetsaraswatpuneet
authored
Basic Scaffolding (#1)
* base nestjs for content publishing * fix build * feedback
1 parent 15a074c commit 390edf4

32 files changed

+11106
-2
lines changed

‎services/content-publishing/.eslintrc.json

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
{
2+
"env": {
3+
"browser": true,
4+
"es2021": true
5+
},
6+
"extends": [
7+
"airbnb-base",
8+
"prettier"
9+
],
10+
"parser": "@typescript-eslint/parser",
11+
"parserOptions": {
12+
"project": "./tsconfig.json",
13+
"sourceType": "module"
14+
},
15+
"settings": {
16+
"import/extensions": [
17+
"error",
18+
"ignorePackages",
19+
{
20+
"js": "never",
21+
"jsx": "never",
22+
"ts": "never",
23+
"tsx": "never"
24+
}
25+
],
26+
"import/parsers": {
27+
"@typescript-eslint/parser": [".ts", ".tsx"]
28+
},
29+
"import/resolver": {
30+
"typescript": {
31+
"directory": "./tsconfig.json"
32+
},
33+
"node": {
34+
"extensions": [".js", ".jsx", ".ts", ".d.ts", ".tsx"]
35+
}
36+
},
37+
"react": {
38+
"version": "999.99.99"
39+
}
40+
},
41+
"rules": {
42+
"no-console": "off",
43+
"import/extensions": [
44+
"error",
45+
"ignorePackages",
46+
{
47+
"js": "never",
48+
"jsx": "never",
49+
"ts": "never",
50+
"tsx": "never"
51+
}
52+
],
53+
"import/no-unresolved": [2, { "commonjs": true, "amd": true }],
54+
"import/named": 2,
55+
"import/namespace": 2,
56+
"import/default": 2,
57+
"import/export": 2,
58+
"import/prefer-default-export": "off",
59+
"indent": "off",
60+
"no-unused-vars": "off",
61+
"prettier/prettier": 2
62+
},
63+
"plugins": ["import", "prettier"]
64+
}

‎services/content-publishing/.github/workflows/build.yml

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
name: Build And Test ContentPublishing Service
2+
concurrency:
3+
group: ${{github.workflow}}-${{github.ref}}
4+
cancel-in-progress: true
5+
on:
6+
pull_request:
7+
branches:
8+
- main
9+
push:
10+
branches:
11+
- main
12+
13+
jobs:
14+
build_Nest_js:
15+
runs-on: ubuntu-latest
16+
steps:
17+
- uses: actions/checkout@v3
18+
- name: Install Node.js
19+
uses: actions/setup-node@v3
20+
with:
21+
node-version: 18
22+
cache: 'npm'
23+
registry-url: 'https://registry.npmjs.org'
24+
cache-dependency-path: package-lock.json
25+
- name: Install dependencies
26+
run: npm ci
27+
- name: Build Nest.js
28+
run: npm run build
29+
test_jest:
30+
runs-on: ubuntu-latest
31+
steps:
32+
- uses: actions/checkout@v3
33+
- name: Install Node.js
34+
uses: actions/setup-node@v3
35+
with:
36+
node-version: 18
37+
cache: 'npm'
38+
registry-url: 'https://registry.npmjs.org'
39+
cache-dependency-path: package-lock.json
40+
- name: Install dependencies
41+
run: npm ci
42+
- name: Run Jest
43+
run: npm run test
44+
check_licenses:
45+
runs-on: ubuntu-latest
46+
steps:
47+
- uses: actions/checkout@v3
48+
- name: Install Node.js
49+
uses: actions/setup-node@v3
50+
with:
51+
node-version: 18
52+
cache: 'npm'
53+
registry-url: 'https://registry.npmjs.org'
54+
cache-dependency-path: package-lock.json
55+
- name: Install dependencies
56+
run: npm ci
57+
- name: License Check
58+
# List all the licenses and error out if it is not one of the supported licenses
59+
run: npx license-report --fields=name --fields=licenseType | jq 'map(select(.licenseType | IN("MIT", "Apache-2.0", "ISC", "BSD-3-Clause", "BSD-2-Clause") | not)) | if length == 0 then halt else halt_error(1) end'

‎services/content-publishing/.github/workflows/common/is-full-release/action.yml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
name: Is Full Release?
2+
description: Determines whether the version tag represents a full release
3+
inputs:
4+
version-tag:
5+
description: "Version tag in v#.#.#[-*] format"
6+
required: true
7+
outputs:
8+
is-full-release:
9+
description: "'true' if full release, 'false' otherwise"
10+
value: ${{steps.is-full-release.outputs.is_full_release}}
11+
runs:
12+
using: "composite"
13+
steps:
14+
- name: Full Release?
15+
id: is-full-release
16+
shell: bash
17+
run: |
18+
version_tag=${{inputs.version-tag}}
19+
is_full_release=$([[ "$version_tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] && \
20+
echo 'true' || echo 'false')
21+
echo "is_full_release: $is_full_release"
22+
echo "is_full_release=$is_full_release" >> $GITHUB_OUTPUT

‎services/content-publishing/.github/workflows/release.yml

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
name: Release
2+
run-name: Cut Release ${{github.event.inputs.release-version || github.ref_name}}
3+
concurrency:
4+
group: ${{github.workflow}}-${{github.ref}}
5+
cancel-in-progress: true
6+
on:
7+
push:
8+
tags:
9+
- 'v[0-9]+.[0-9]+.[0-9]+' # ex. v1.0.0
10+
- 'v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+' # ex. v1.1.0-rc1
11+
- 'v0.0.1' # used for testing only
12+
- 'v0.0.1-rc[0-9]+' # used for testing only
13+
workflow_dispatch:
14+
inputs:
15+
release-version:
16+
description: 'Release version (v#.#.#[-rc#])'
17+
required: true
18+
19+
env:
20+
NEW_RELEASE_TAG_FROM_UI: ${{github.event.inputs.release-version}}
21+
TEST_RUN: ${{startsWith(github.event.inputs.release-version || github.ref_name, 'v0.0.1')}}
22+
DOCKER_HUB_PROFILE: amplicalabs
23+
IMAGE_NAME: content-publishing-service
24+
25+
jobs:
26+
build-and-publish-container-image:
27+
name: Build and publish container image
28+
runs-on: ubuntu-20.04
29+
container: ghcr.io/libertydsnp/frequency/ci-base-image
30+
steps:
31+
- name: Validate Version Tag
32+
if: env.NEW_RELEASE_TAG_FROM_UI != ''
33+
shell: bash
34+
run: |
35+
version=${{env.NEW_RELEASE_TAG_FROM_UI}}
36+
echo "Release version entered in UI: $version"
37+
regex='^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-rc[1-9]\d*)?$'
38+
if [[ ! $version =~ $regex ]]; then
39+
echo "ERROR: Entered version $version is not valid."
40+
echo "Please use v#.#.#[-rc#] format."
41+
exit 1
42+
fi
43+
echo "valid-version=true" >> $GITHUB_OUTPUT
44+
- name: Check Out Repo
45+
uses: actions/checkout@v3
46+
with:
47+
ref: ${{env.NEW_RELEASE_TAG_FROM_UI}}
48+
- name: Set up tags for standalone image
49+
id: standalone-tags
50+
uses: docker/metadata-action@v4
51+
with:
52+
flavor: |
53+
latest=auto
54+
prefix=standalone-,onlatest=true
55+
images: |
56+
${{env.DOCKER_HUB_PROFILE}}/${{env.IMAGE_NAME}}
57+
tags: |
58+
type=semver,pattern={{version}}
59+
- name: Set up tags for app-only image
60+
id: app-only-tags
61+
uses: docker/metadata-action@v4
62+
with:
63+
flavor: |
64+
latest=auto
65+
prefix=apponly-,onlatest=true
66+
images: |
67+
${{env.DOCKER_HUB_PROFILE}}/${{env.IMAGE_NAME}}
68+
tags: |
69+
type=semver,pattern={{version}}
70+
- name: Set up QEMU
71+
uses: docker/setup-qemu-action@v2
72+
with:
73+
platforms: |
74+
linux/amd64
75+
- name: Set up Docker Buildx
76+
uses: docker/setup-buildx-action@v2
77+
- name: Login to DockerHub
78+
uses: docker/login-action@v2
79+
with:
80+
username: ${{secrets.DOCKERHUB_USERNAME_FC}}
81+
password: ${{secrets.DOCKERHUB_TOKEN_FC}}
82+
- name: Build and Push Standalone (Complete) Container Image
83+
uses: docker/build-push-action@v4
84+
with:
85+
context: .
86+
platforms: linux/amd64
87+
push: ${{env.TEST_RUN != 'true'}}
88+
file: ./Dockerfile
89+
target: standalone
90+
tags: ${{ steps.standalone-tags.outputs.tags }}
91+
- name: Build and Push App-Only Container Image
92+
uses: docker/build-push-action@v4
93+
with:
94+
context: .
95+
platforms: linux/amd64
96+
push: ${{env.TEST_RUN != 'true'}}
97+
file: ./Dockerfile
98+
target: app-only
99+
tags: ${{ steps.app-only-tags.outputs.tags }}

‎services/content-publishing/.gitignore

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
node_modules
2+
dist
3+
.env*
4+
.vscode
5+
coverage

‎services/content-publishing/.prettierrc

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{
2+
"arrowParens": "always",
3+
"bracketSpacing": true,
4+
"semi": true,
5+
"trailingComma": "all",
6+
"singleQuote": true,
7+
"printWidth": 180,
8+
"tabWidth": 2,
9+
"useTabs": false
10+
}

‎services/content-publishing/.tool-versions

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
nodejs 18.16.0

‎services/content-publishing/Dockerfile

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
FROM --platform=linux/amd64 node:18 as build
2+
3+
# TODO: The deployment docker image should install the content publishing
4+
# service from NPM rather than building from source
5+
WORKDIR /app
6+
COPY package*.json ./
7+
RUN npm install
8+
9+
# Build / Copy the rest of the application files to the container and build
10+
COPY . .
11+
RUN npm run build
12+
13+
FROM build as app-only
14+
15+
EXPOSE 3000
16+
17+
ENTRYPOINT npm start
18+
19+
FROM build as standalone
20+
21+
# Install Redis on top of the base image
22+
RUN apt-get -y update
23+
RUN apt-get -y install redis
24+
RUN sed -e 's/^appendonly .*$/appendonly yes/' /etc/redis/redis.conf > /etc/redis/redis.conf.appendonly
25+
RUN mv /etc/redis/redis.conf.appendonly /etc/redis/redis.conf
26+
27+
ENV REDIS_URL=redis://localhost:6379
28+
29+
VOLUME [ "/var/lib/redis" ]
30+
31+
# Start the application
32+
ENTRYPOINT service redis-server start && npm start

‎services/content-publishing/INSTALLING.md

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
# INSTALLING
2+
3+
## A Note about Redis Persistence
4+
5+
The application requires a Redis server that is configured with `Append-only file` persistence. This is so that application state can be maintained across Redis restarts. Notes on how to configure this are included below for each type of deployment.
6+
7+
8+
## Deploying using prebuilt Docker images
9+
10+
### Standalone (complete) image
11+
12+
The standalone container image is meant to be a complete solution for a provider. It contains a single instance of the main application, plus a pre-configured Redis server. Simply download the latest [container image](https://hub.docker.com/r/amplicalabs/content-publishing-service/) and deploy using your favorite container management system.
13+
```
14+
docker pull amplicalabs/content-publishing-service:standalone-latest
15+
```
16+
17+
The internal Redis server included in the complete image is already configured for persistence; it is simply necessary to configure your container pod to map the directory `/var/lib/redis` to a persistent storage volume.
18+
19+
#### Note: The internal redis server runs as user:group 100:102, so mapped volume permissions must at minimum allow write access to this user. How this is provisioned will depend on the specifics of your persistent storage infrastructure. If this is not configured properly, the redis server will fail to start, and the application upon launch will throw `ECONNREFUSED` errors.
20+
21+
Follow the instructions below for [configuration](#configuration), with the exception that you should _not_ modify `REDIS_URL`, as it already points to the internal Redis server.
22+
23+
### App-only image
24+
25+
The app-only image is meant to be used for providers who would rather utilize a Redis instance in their own (or their cloud infrastructure provider's) external Redis instance or service. To download the latest [container image](https://hub.docker.com/r/amplicalabs/content-publishing-service/), simply:
26+
```
27+
docker pull amplicalabs/content-publishing-service:apponly-latest
28+
```
29+
In this case, you need to ensure that the following settings are configured in your Redis instance:
30+
```
31+
appendonly true
32+
dir <base directory for Redis storage>
33+
appendonlydir <subdirectory of base directory where append-only persistence files are stored>
34+
```
35+
36+
You must also minimally map `appendonlydir` (or the entire `dir`) to a persistent storage volume in your infrastructure environment
37+
38+
## Building and Deploying the Application
39+
40+
If you choose to build & deploy the application yourself, simply install the prerequisites:
41+
* NodeJS 18
42+
43+
Note that, at present, due to limitaions in the `@dsnp/graph-sdk` module, the applcation can only be run on a `linux/amd64` platform. Support for other platforms is contingent on building & installing the `@dsnp/graph-sdk` module from source, which is outside the scope of this document.
44+
45+
To build the application:
46+
```
47+
npm run build
48+
```
49+
50+
To run the application:
51+
```
52+
npm start
53+
```
54+
55+
## Configuration
56+
57+
For the application to start & run correctly, it is necessary to configure the environment with certain parameters. These should be injected into a container pod if running in a containerized environment.
58+
59+
The following is a list of environment variables that may be set to control the application's behavior and environment. The complete list can always be referenced [here](./env.template)
60+
61+
|Variable|required?|Description|Default|
62+
|-|-|-|-|
63+
|`FREQUENCY_URL`|**yes**|Blockchain URL|_none_|
64+
|`PROVIDER_ID`|**yes**|MSA ID of provider|_none_|
65+
|`PROVIDER_BASE_URL`|**yes**|URL of provider graph query endpoint|_none_|
66+
|`PROVIDER_ACCESS_TOKEN`|no|Optional access token to be used with requests to provider graph query endpoint|_none_|
67+
|`PROVIDER_ACCOUNT_SEED_PHRASE`|**yes**|Seed phrase for provider control keypair|_none_|
68+
|`REDIS_URL`|**yes**|URL used to connect to Redis instance|_none_<br/>\*preset to the internal Redis URL in the standalone container|
69+
|`BLOCKCHAIN_SCAN_INTERVAL_MINUTES`|no|# of minutes to wait in between scans of the blockchain|180|
70+
|`QUEUE_HIGH_WATER`|no|# of pending graph scan queue entries to allow before pausing blockchain scanning until the next scan cycle|1000|
71+
|`GRAPH_ENVIRONMENT_TYPE`|**yes**|Indicates which blockchain network to connect to.<br/>Possible values:<br/>* `Mainnet`<br/>* `Rococo`<br/>* `Dev`|_none_|
72+
|`GRAPH_ENVIRONMENT_DEV_CONFIG`|no<br/>*required for 'Dev' Graph Environment|JSON configuration object for GraphSDK configuration. Used to test against a local development Frequency node|_none_|

‎services/content-publishing/README.md

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
1-
# content-publishing-service
2-
A microservice to publish DSNP content to frequency
1+
# Content Publisher
2+
3+
A microservice to publish DSNP content to frequency.

‎services/content-publishing/dev.Dockerfile

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
FROM node:18-alpine3.17
2+
3+
WORKDIR /app
4+
5+
# Start the application
6+
CMD ["npm", "run", "start:dev:docker"]

0 commit comments

Comments
 (0)