Skip to content

Commit be5b87f

Browse files
saraswatpuneetsaraswatpuneet
authored
Docker compose with frq,redis,ipfs (#81)
* docker compose with frq,redis,ipfs * add docker env * finalize * add simple procedure to readme * fix * fix env file for docker compose * env file for docker, update docker files * cleanup * no need for chain names, we are chain agnostic * cleanup * add missing port * worker dont need to expose ports * everyone gets a volume
1 parent 5af8f6c commit be5b87f

File tree

12 files changed

+240
-125
lines changed

12 files changed

+240
-125
lines changed

‎services/content-watcher/.dockerignore

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
Dockerfile
2+
.dockerignore
3+
node_modules
4+
npm-debug.log
5+
dist
6+
.env*
7+
env.template

‎services/content-watcher/.env.docker.dev

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Copy this file to ".env.dev" and ".env.docker.dev", and then tweak values for local development
2+
# IPFS_ENDPOINT="https://ipfs.infura.io:5001"
3+
# IPFS_BASIC_AUTH_USER="Infura Project ID Here or Blank for Kubo RPC"
4+
# IPFS_BASIC_AUTH_SECRET="Infura Secret Here or Blank for Kubo RPC"
5+
# IPFS_GATEWAY_URL="https://ipfs.io/ipfs/[CID]"
6+
IPFS_ENDPOINT="http://kubo_ipfs:5001"
7+
IPFS_BASIC_AUTH_USER=""
8+
IPFS_BASIC_AUTH_SECRET=""
9+
IPFS_GATEWAY_URL="http://kubo_ipfs:8080/ipfs/[CID]"
10+
11+
FREQUENCY_URL=ws://frequency:9944
12+
PROVIDER_ID=1
13+
REDIS_URL=redis://redis:6379
14+
BLOCKCHAIN_SCAN_INTERVAL_MINUTES=1
15+
QUEUE_HIGH_WATER=1000
16+
PROVIDER_ACCOUNT_SEED_PHRASE="//Ferdie"
17+
WEBHOOK_FAILURE_THRESHOLD=3
18+
HEALTH_CHECK_SUCCESS_THRESHOLD=10
19+
WEBHOOK_RETRY_INTERVAL_SECONDS=10
20+
HEALTH_CHECK_MAX_RETRY_INTERVAL_SECONDS=10
21+
HEALTH_CHECK_MAX_RETRIES=4
22+
CAPACITY_LIMIT='{"type":"percentage", "value":80}'
23+
ENVIRONMENT="dev"
24+
API_PORT=3000
25+
26+
FILE_UPLOAD_MAX_SIZE_IN_BYTES=2000000000
27+
ASSET_EXPIRATION_INTERVAL_SECONDS=300
28+
BATCH_INTERVAL_SECONDS=12
29+
BATCH_MAX_COUNT=1000
30+
ASSET_UPLOAD_VERIFICATION_DELAY_SECONDS=5

‎services/content-watcher/.github/workflows/release.yml

Lines changed: 6 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ env:
2020
NEW_RELEASE_TAG_FROM_UI: ${{github.event.inputs.release-version}}
2121
TEST_RUN: ${{startsWith(github.event.inputs.release-version || github.ref_name, 'v0.0.1')}}
2222
DOCKER_HUB_PROFILE: amplicalabs
23-
IMAGE_NAME: content-publishing-service
23+
IMAGE_NAME: content-publishing-service-api
2424

2525
jobs:
2626
build-and-publish-container-image:
@@ -45,24 +45,13 @@ jobs:
4545
uses: actions/checkout@v3
4646
with:
4747
ref: ${{env.NEW_RELEASE_TAG_FROM_UI}}
48-
- name: Set up tags for standalone image
49-
id: standalone-tags
48+
- name: Set up tags for cp image
49+
id: cp-tags
5050
uses: docker/metadata-action@v4
5151
with:
5252
flavor: |
5353
latest=auto
54-
prefix=standalone-,onlatest=true
55-
images: |
56-
${{env.DOCKER_HUB_PROFILE}}/${{env.IMAGE_NAME}}
57-
tags: |
58-
type=semver,pattern={{version}}
59-
- name: Set up tags for app-only image
60-
id: app-only-tags
61-
uses: docker/metadata-action@v4
62-
with:
63-
flavor: |
64-
latest=auto
65-
prefix=apponly-,onlatest=true
54+
prefix=api-,onlatest=true
6655
images: |
6756
${{env.DOCKER_HUB_PROFILE}}/${{env.IMAGE_NAME}}
6857
tags: |
@@ -79,21 +68,11 @@ jobs:
7968
with:
8069
username: ${{secrets.DOCKERHUB_USERNAME_FC}}
8170
password: ${{secrets.DOCKERHUB_TOKEN_FC}}
82-
- name: Build and Push Standalone (Complete) Container Image
83-
uses: docker/build-push-action@v4
84-
with:
85-
context: .
86-
platforms: linux/amd64
87-
push: ${{env.TEST_RUN != 'true'}}
88-
file: ./Dockerfile
89-
target: standalone
90-
tags: ${{ steps.standalone-tags.outputs.tags }}
91-
- name: Build and Push App-Only Container Image
71+
- name: Build and Push Content-Publishing-Service Image
9272
uses: docker/build-push-action@v4
9373
with:
9474
context: .
9575
platforms: linux/amd64
9676
push: ${{env.TEST_RUN != 'true'}}
9777
file: ./Dockerfile
98-
target: app-only
99-
tags: ${{ steps.app-only-tags.outputs.tags }}
78+
tags: ${{ steps.cp-tags.outputs.tags }}

‎services/content-watcher/.gitignore

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
node_modules
22
dist
3-
.env*
3+
.env
44
.vscode
55
coverage
66
.idea

‎services/content-watcher/Dockerfile

Lines changed: 16 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,28 @@
1-
FROM --platform=linux/amd64 node:18 as build
1+
# Use a multi-stage build for efficiency
2+
FROM node:18 AS builder
3+
4+
WORKDIR /usr/src/app
25

3-
# TODO: The deployment docker image should install the content publishing
4-
# service from NPM rather than building from source
5-
WORKDIR /app
66
COPY package*.json ./
7+
78
RUN npm install
89

9-
# Build / Copy the rest of the application files to the container and build
1010
COPY . .
11-
RUN npm run build
12-
13-
FROM build as app-only
1411

15-
EXPOSE 3000
12+
# Build the application
13+
RUN npm run build
1614

17-
ENTRYPOINT npm start
15+
# Production stage
16+
FROM node:18
1817

19-
FROM build as standalone
18+
WORKDIR /usr/src/app
2019

21-
# Install Redis on top of the base image
22-
RUN apt-get -y update
23-
RUN apt-get -y install redis
24-
RUN sed -e 's/^appendonly .*$/appendonly yes/' /etc/redis/redis.conf > /etc/redis/redis.conf.appendonly
25-
RUN mv /etc/redis/redis.conf.appendonly /etc/redis/redis.conf
20+
COPY --from=builder /usr/src/app/dist ./dist
21+
COPY package*.json ./
2622

27-
ENV REDIS_URL=redis://localhost:6379
23+
RUN npm install --only=production
24+
EXPOSE 3000
25+
ENV START_PROCESS="api"
2826

29-
VOLUME [ "/var/lib/redis" ]
3027

31-
# Start the application
32-
ENTRYPOINT service redis-server start && npm start
28+
CMD ["sh", "-c", "if [ \"$START_PROCESS\" = \"api\" ]; then npm run start:api:prod; else npm run start:worker:prod; fi"]

‎services/content-watcher/README.md

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,41 @@
11
# Content Publisher
22

3-
A microservice to publish DSNP content to frequency.
3+
Content Publisher is a microservice designed to publish DSNP (Decentralized Social Networking Protocol) content to the Frequency blockchain. This README provides step-by-step instructions to set up and run the service.
4+
5+
## Table of Contents
6+
7+
- [Content Publisher](#content-publisher)
8+
- [Table of Contents](#table-of-contents)
9+
- [Prerequisites](#prerequisites)
10+
- [Getting Started](#getting-started)
11+
- [Clone the Repository](#clone-the-repository)
12+
13+
## Prerequisites
14+
15+
Before you begin, ensure you have met the following requirements:
16+
17+
- **Docker:** Content Publisher is designed to run in a Docker environment. Make sure Docker is installed on your system.
18+
19+
## Getting Started
20+
21+
Follow these steps to set up and run Content Publisher:
22+
23+
### Clone the Repository
24+
25+
1. Clone the Content Publisher repository to your local machine:
26+
27+
```bash
28+
git clone https://github.com/amplicalabls/content-publishing-service.git
29+
```
30+
31+
2. Modify any environment variables in the `.env` file as needed. For docker compose env `.env.docker.dev` file is used.
32+
33+
3. Run the following command to start the service:
34+
35+
```bash
36+
docker-compose -f docker-compose.dev.yaml up
37+
```
38+
39+
4. Visit [Swagger UI](http://localhost:3000/api/docs/swagger) to view the API documentation and submit requests to the service.
40+
41+
5. Visit [Bullboard](http://localhost:3000/queues) to view the job queue and the status of the jobs.

‎services/content-watcher/apps/api/src/api.module.ts

Lines changed: 72 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -16,52 +16,84 @@ import { ConfigService } from '../../../libs/common/src/config/config.service';
1616

1717
@Module({
1818
imports: [
19-
BullModule.forRoot({
20-
connection: {
21-
enableOfflineQueue: false,
19+
ConfigModule,
20+
RedisModule.forRootAsync(
21+
{
22+
imports: [ConfigModule],
23+
useFactory: (configService: ConfigService) => ({
24+
config: [{ url: configService.redisUrl.toString() }],
25+
}),
26+
inject: [ConfigService],
2227
},
28+
true, // isGlobal
29+
),
30+
BullModule.forRootAsync({
31+
imports: [ConfigModule],
32+
useFactory: (configService: ConfigService) => {
33+
// Note: BullMQ doesn't honor a URL for the Redis connection, and
34+
// JS URL doesn't parse 'redis://' as a valid protocol, so we fool
35+
// it by changing the URL to use 'http://' in order to parse out
36+
// the host, port, username, password, etc.
37+
// We could pass REDIS_HOST, REDIS_PORT, etc, in the environment, but
38+
// trying to keep the # of environment variables from proliferating
39+
const url = new URL(configService.redisUrl.toString().replace(/^redis[s]*/, 'http'));
40+
const { hostname, port, username, password, pathname } = url;
41+
return {
42+
connection: {
43+
host: hostname || undefined,
44+
port: port ? Number(port) : undefined,
45+
username: username || undefined,
46+
password: password || undefined,
47+
db: pathname?.length > 1 ? Number(pathname.slice(1)) : undefined,
48+
},
49+
};
50+
},
51+
inject: [ConfigService],
2352
}),
53+
BullModule.registerQueue(
54+
{
55+
name: QueueConstants.ASSET_QUEUE_NAME,
56+
},
57+
{
58+
name: QueueConstants.REQUEST_QUEUE_NAME,
59+
},
60+
{
61+
name: QueueConstants.BROADCAST_QUEUE_NAME,
62+
},
63+
{
64+
name: QueueConstants.REPLY_QUEUE_NAME,
65+
},
66+
{
67+
name: QueueConstants.REACTION_QUEUE_NAME,
68+
},
69+
{
70+
name: QueueConstants.TOMBSTONE_QUEUE_NAME,
71+
},
72+
{
73+
name: QueueConstants.UPDATE_QUEUE_NAME,
74+
},
75+
{
76+
name: QueueConstants.PROFILE_QUEUE_NAME,
77+
},
78+
{
79+
name: QueueConstants.BATCH_QUEUE_NAME,
80+
},
81+
{
82+
name: QueueConstants.PUBLISH_QUEUE_NAME,
83+
},
84+
{
85+
name: QueueConstants.TRANSACTION_RECEIPT_QUEUE_NAME,
86+
},
87+
{
88+
name: QueueConstants.STATUS_QUEUE_NAME,
89+
},
90+
),
91+
92+
// Bullboard UI
2493
BullBoardModule.forRoot({
2594
route: '/queues',
2695
adapter: ExpressAdapter,
2796
}),
28-
BullModule.registerQueue({
29-
name: QueueConstants.ASSET_QUEUE_NAME,
30-
}),
31-
BullModule.registerQueue({
32-
name: QueueConstants.REQUEST_QUEUE_NAME,
33-
}),
34-
BullModule.registerQueue({
35-
name: QueueConstants.BROADCAST_QUEUE_NAME,
36-
}),
37-
BullModule.registerQueue({
38-
name: QueueConstants.REPLY_QUEUE_NAME,
39-
}),
40-
BullModule.registerQueue({
41-
name: QueueConstants.REACTION_QUEUE_NAME,
42-
}),
43-
BullModule.registerQueue({
44-
name: QueueConstants.TOMBSTONE_QUEUE_NAME,
45-
}),
46-
BullModule.registerQueue({
47-
name: QueueConstants.UPDATE_QUEUE_NAME,
48-
}),
49-
BullModule.registerQueue({
50-
name: QueueConstants.PROFILE_QUEUE_NAME,
51-
}),
52-
BullModule.registerQueue({
53-
name: QueueConstants.BATCH_QUEUE_NAME,
54-
}),
55-
BullModule.registerQueue({
56-
name: QueueConstants.PUBLISH_QUEUE_NAME,
57-
}),
58-
BullModule.registerQueue({
59-
name: QueueConstants.TRANSACTION_RECEIPT_QUEUE_NAME,
60-
}),
61-
BullModule.registerQueue({
62-
name: QueueConstants.STATUS_QUEUE_NAME,
63-
}),
64-
6597
BullBoardModule.forFeature({
6698
name: QueueConstants.ASSET_QUEUE_NAME,
6799
adapter: BullMQAdapter,
@@ -110,17 +142,6 @@ import { ConfigService } from '../../../libs/common/src/config/config.service';
110142
name: QueueConstants.STATUS_QUEUE_NAME,
111143
adapter: BullMQAdapter,
112144
}),
113-
ConfigModule,
114-
RedisModule.forRootAsync(
115-
{
116-
imports: [ConfigModule],
117-
useFactory: (configService: ConfigService) => ({
118-
config: [{ url: configService.redisUrl.toString() }],
119-
}),
120-
inject: [ConfigService],
121-
},
122-
true, // isGlobal
123-
),
124145
EventEmitterModule.forRoot({
125146
// Use this instance throughout the application
126147
global: true,

‎services/content-watcher/dev.Dockerfile

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
1-
FROM node:18-alpine3.17
1+
FROM node:18
22

33
WORKDIR /app
4+
COPY . .
45

5-
# Start the application
6-
CMD ["npm", "run", "start:dev:docker"]
6+
RUN npm install
7+
EXPOSE 3000
8+
ENV START_PROCESS="api"
9+
10+
CMD ["sh", "-c", "if [ \"$START_PROCESS\" = \"api\" ]; then npm run start:api; else npm run start:worker; fi"]

0 commit comments

Comments
 (0)