Merge pull request #5 from AmplicaLabs/setup_repo

[Chore] setup repository with relevant code

Author: saraswatpuneet

services/content-watcher/.github/workflows/release.yml

@@ -20,7 +20,7 @@ env:
   NEW_RELEASE_TAG_FROM_UI: ${{github.event.inputs.release-version}}
   TEST_RUN: ${{startsWith(github.event.inputs.release-version || github.ref_name, 'v0.0.1')}}
   DOCKER_HUB_PROFILE: amplicalabs
-  IMAGE_NAME: content-publishing-service-api
+  IMAGE_NAME: content-watcher-service
 
 jobs:
   build-and-publish-container-image:
@@ -68,7 +68,7 @@ jobs:
         with:
           username: ${{secrets.DOCKERHUB_USERNAME_FC}}
           password: ${{secrets.DOCKERHUB_TOKEN_FC}}
-      - name: Build and Push Content-Publishing-Service Image
+      - name: Build and Push content-watcher-service Image
         uses: docker/build-push-action@v4
         with:
           context: .

services/content-watcher/INSTALLING.md

@@ -9,9 +9,9 @@ The application requires a Redis server that is configured with `Append-only fil
 
 ### Standalone (complete) image
 
-The standalone container image is meant to be a complete solution for a provider. It contains a single instance of the main application, plus a pre-configured Redis server. Simply download the latest [container image](https://hub.docker.com/r/amplicalabs/content-publishing-service/) and deploy using your favorite container management system.
+The standalone container image is meant to be a complete solution for a provider. It contains a single instance of the main application, plus a pre-configured Redis server. Simply download the latest [container image](https://hub.docker.com/r/amplicalabs/content-watcher-service/) and deploy using your favorite container management system.
 ```
-    docker pull amplicalabs/content-publishing-service:standalone-latest
+    docker pull amplicalabs/content-watcher-service:standalone-latest
 ```
 
 The internal Redis server included in the complete image is already configured for persistence; it is simply necessary to configure your container pod to map the directory `/var/lib/redis` to a persistent storage volume.
@@ -22,9 +22,9 @@ Follow the instructions below for [configuration](#configuration), with the exce
 
 ### App-only image
 
-The app-only image is meant to be used for providers who would rather utilize a Redis instance in their own (or their cloud infrastructure provider's) external Redis instance or service. To download the latest [container image](https://hub.docker.com/r/amplicalabs/content-publishing-service/), simply:
+The app-only image is meant to be used for providers who would rather utilize a Redis instance in their own (or their cloud infrastructure provider's) external Redis instance or service. To download the latest [container image](https://hub.docker.com/r/amplicalabs/content-watcher-service/), simply:
 ```
-    docker pull amplicalabs/content-publishing-service:apponly-latest
+    docker pull amplicalabs/content-watcher-service:apponly-latest
 ```
 In this case, you need to ensure that the following settings are configured in your Redis instance:
 ```

services/content-watcher/README.md

@@ -25,7 +25,7 @@ Follow these steps to set up and run Content Publisher:
 1. Clone the Content Publisher repository to your local machine:
 
    ```bash
-   git clone https://github.com/amplicalabls/content-publishing-service.git
+   git clone https://github.com/amplicalabls/content-watcher-service.git
    ```
 
 2. Modify any environment variables in the `.env` file as needed. For docker compose env `.env.docker.dev` file is used.

services/content-watcher/apps/api/src/api.controller.ts

@@ -33,73 +33,4 @@ export class ApiController {
       status: HttpStatus.OK,
     };
   }
-
-  @Put('asset/upload')
-  @UseInterceptors(FilesInterceptor('files'))
-  @HttpCode(202)
-  @ApiConsumes('multipart/form-data')
-  @ApiBody({
-    description: 'Asset files',
-    type: FilesUploadDto,
-  })
-  async assetUpload(
-    @UploadedFiles(
-      new ParseFilePipeBuilder()
-        .addFileTypeValidator({
-          fileType: DSNP_VALID_MIME_TYPES,
-        })
-        // TODO: add a validator to check overall uploaded size
-        .addMaxSizeValidator({
-          // this is in bytes (2 GB)
-          maxSize: parseInt(process.env.FILE_UPLOAD_MAX_SIZE_IN_BYTES!, 10),
-        })
-        .build({
-          errorHttpStatusCode: HttpStatus.UNPROCESSABLE_ENTITY,
-        }),
-    )
-    files: // eslint-disable-next-line no-undef
-    Array<Express.Multer.File>,
-  ): Promise<UploadResponseDto> {
-    return this.apiService.addAssets(files);
-  }
-
-  @Post('content/:userDsnpId/broadcast')
-  @HttpCode(202)
-  async broadcast(@Param() userDsnpId: DsnpUserIdParam, @Body() broadcastDto: BroadcastDto): Promise<AnnouncementResponseDto> {
-    const metadata = await this.apiService.validateAssetsAndFetchMetadata(broadcastDto as AssetIncludedRequestDto);
-    return this.apiService.enqueueRequest(AnnouncementTypeDto.BROADCAST, userDsnpId.userDsnpId, broadcastDto, metadata);
-  }
-
-  @Post('content/:userDsnpId/reply')
-  @HttpCode(202)
-  async reply(@Param() userDsnpId: DsnpUserIdParam, @Body() replyDto: ReplyDto): Promise<AnnouncementResponseDto> {
-    const metadata = await this.apiService.validateAssetsAndFetchMetadata(replyDto as AssetIncludedRequestDto);
-    return this.apiService.enqueueRequest(AnnouncementTypeDto.REPLY, userDsnpId.userDsnpId, replyDto, metadata);
-  }
-
-  @Post('content/:userDsnpId/reaction')
-  @HttpCode(202)
-  async reaction(@Param() userDsnpId: DsnpUserIdParam, @Body() reactionDto: ReactionDto): Promise<AnnouncementResponseDto> {
-    return this.apiService.enqueueRequest(AnnouncementTypeDto.REACTION, userDsnpId.userDsnpId, reactionDto);
-  }
-
-  @Put('content/:userDsnpId')
-  @HttpCode(202)
-  async update(@Param() userDsnpId: DsnpUserIdParam, @Body() updateDto: UpdateDto): Promise<AnnouncementResponseDto> {
-    const metadata = await this.apiService.validateAssetsAndFetchMetadata(updateDto as AssetIncludedRequestDto);
-    return this.apiService.enqueueRequest(AnnouncementTypeDto.UPDATE, userDsnpId.userDsnpId, updateDto, metadata);
-  }
-
-  @Delete('content/:userDsnpId')
-  @HttpCode(202)
-  async delete(@Param() userDsnpId: DsnpUserIdParam, @Body() tombstoneDto: TombstoneDto): Promise<AnnouncementResponseDto> {
-    return this.apiService.enqueueRequest(AnnouncementTypeDto.TOMBSTONE, userDsnpId.userDsnpId, tombstoneDto);
-  }
-
-  @Put('profile/:userDsnpId')
-  @HttpCode(202)
-  async profile(@Param() userDsnpId: DsnpUserIdParam, @Body() profileDto: ProfileDto): Promise<AnnouncementResponseDto> {
-    const metadata = await this.apiService.validateAssetsAndFetchMetadata(profileDto as AssetIncludedRequestDto);
-    return this.apiService.enqueueRequest(AnnouncementTypeDto.PROFILE, userDsnpId.userDsnpId, profileDto, metadata);
-  }
 }

services/content-watcher/apps/api/src/api.module.ts

@@ -7,7 +7,6 @@ import { BullBoardModule } from '@bull-board/nestjs';
 import { BullMQAdapter } from '@bull-board/api/bullMQAdapter';
 import { ExpressAdapter } from '@bull-board/express';
 import { ApiController } from './api.controller';
-import { DevelopmentController } from './development.controller';
 import { QueueConstants } from '../../../libs/common/src';
 import { ApiService } from './api.service';
 import { IpfsService } from '../../../libs/common/src/utils/ipfs.client';
@@ -163,7 +162,7 @@ import { ConfigService } from '../../../libs/common/src/config/config.service';
     ScheduleModule.forRoot(),
   ],
   providers: [ConfigService, ApiService, IpfsService],
-  controllers: process.env?.ENVIRONMENT === 'dev' ? [DevelopmentController, ApiController] : [ApiController],
+  controllers: [ApiController],
   exports: [],
 })
 export class ApiModule {}

services/content-watcher/apps/api/src/api.service.ts

@@ -19,8 +19,6 @@ import {
 import { calculateIpfsCID } from '../../../libs/common/src/utils/ipfs';
 import { IAssetJob, IAssetMetadata } from '../../../libs/common/src/interfaces/asset-job.interface';
 import { RedisUtils } from '../../../libs/common/src/utils/redis';
-import getAssetDataKey = RedisUtils.getAssetDataKey;
-import getAssetMetadataKey = RedisUtils.getAssetMetadataKey;
 
 @Injectable()
 export class ApiService {
@@ -34,134 +32,6 @@ export class ApiService {
     this.logger = new Logger(this.constructor.name);
   }
 
-  async enqueueRequest(
-    announcementType: AnnouncementTypeDto,
-    dsnpUserId: string,
-    content: RequestTypeDto,
-    assetToMimeType?: Map<string, string>,
-  ): Promise<AnnouncementResponseDto> {
-    const data = {
-      content,
-      id: '',
-      announcementType,
-      dsnpUserId,
-      dependencyAttempt: 0,
-    } as IRequestJob;
-    data.id = this.calculateJobId(data);
-    if (assetToMimeType) {
-      // not used in id calculation since the order in map might not be deterministic
-      data.assetToMimeType = assetToMimeType;
-    }
-    const job = await this.requestQueue.add(`Request Job - ${data.id}`, data, { jobId: data.id, removeOnFail: false, removeOnComplete: 2000 }); // TODO: should come from queue configs
-    this.logger.debug(job);
-    return {
-      referenceId: data.id,
-    };
-  }
-
-  async validateAssetsAndFetchMetadata(content: AssetIncludedRequestDto): Promise<Map<string, string> | undefined> {
-    const checkingList: Array<{ onlyImage: boolean; referenceId: string }> = [];
-    if (content.profile) {
-      content.profile.icon?.forEach((reference) => checkingList.push({ onlyImage: true, referenceId: reference.referenceId }));
-    } else if (content.content) {
-      content.content.assets?.forEach(
-        (asset) =>
-          asset.references?.forEach((reference) =>
-            checkingList.push({
-              onlyImage: false,
-              referenceId: reference.referenceId,
-            }),
-          ),
-      );
-    }
-
-    const redisResults = await Promise.all(checkingList.map((obj) => this.redis.get(getAssetMetadataKey(obj.referenceId))));
-    const errors: string[] = [];
-    const map = new Map();
-    redisResults.forEach((res, index) => {
-      if (res === null) {
-        errors.push(`${content.profile ? 'profile.icon' : 'content.assets'}.referenceId ${checkingList[index].referenceId} does not exist!`);
-      } else {
-        const metadata: IAssetMetadata = JSON.parse(res);
-        map[checkingList[index].referenceId] = metadata.mimeType;
-
-        // checks if attached asset is an image
-        if (checkingList[index].onlyImage && !isImage(metadata.mimeType)) {
-          errors.push(`profile.icon.referenceId ${checkingList[index].referenceId} is not an image!`);
-        }
-      }
-    });
-    if (errors.length > 0) {
-      throw new HttpErrorByCode[400](errors);
-    }
-    return map;
-  }
-
-  // eslint-disable-next-line no-undef
-  async addAssets(files: Array<Express.Multer.File>): Promise<UploadResponseDto> {
-    // calculate ipfs cid references
-    const referencePromises: Promise<string>[] = files.map((file) => calculateIpfsCID(file.buffer));
-    const references = await Promise.all(referencePromises);
-
-    let dataTransaction = this.redis.multi();
-    let metadataTransaction = this.redis.multi();
-    const jobs: any[] = [];
-    files.forEach((f, index) => {
-      // adding data and metadata to the transaction
-      dataTransaction = dataTransaction.setex(getAssetDataKey(references[index]), RedisUtils.STORAGE_EXPIRE_UPPER_LIMIT_SECONDS, f.buffer);
-      metadataTransaction = metadataTransaction.setex(
-        getAssetMetadataKey(references[index]),
-        RedisUtils.STORAGE_EXPIRE_UPPER_LIMIT_SECONDS,
-        JSON.stringify({
-          ipfsCid: references[index],
-          mimeType: f.mimetype,
-          createdOn: Date.now(),
-        } as IAssetMetadata),
-      );
-      // adding asset job to the jobs
-      jobs.push({
-        name: `Asset Job - ${references[index]}`,
-        data: {
-          ipfsCid: references[index],
-          contentLocation: getAssetDataKey(references[index]),
-          metadataLocation: getAssetMetadataKey(references[index]),
-          mimeType: f.mimetype,
-        } as IAssetJob,
-        opts: {
-          jobId: references[index],
-          removeOnFail: false,
-          removeOnComplete: true,
-          attempts: 3,
-          backoff: {
-            type: 'exponential',
-            delay: 10000,
-          },
-        } as BulkJobOptions,
-      });
-    });
-
-    // currently we are applying 3 different transactions on redis
-    // 1: Storing the content data
-    // 2: Adding asset jobs
-    // 3: Storing the content metadata
-    // even though all these transactions are applied separately, the overall behavior will clean up any partial failures eventually
-    // partial failure scenarios:
-    //    1: adding jobs failure: at this point we already successfully stored the data content in redis, but since all
-    //       of this stored data has expire-time, it would eventually get cleaned up
-    //    2: metadata transaction failure: at this point we already stored the data content and jobs and those two are
-    //       enough to process the asset on the worker side, the worker will clean up both of them after processing
-    const dataOps = await dataTransaction.exec();
-    this.checkTransactionResult(dataOps);
-    const queuedJobs = await this.assetQueue.addBulk(jobs);
-    this.logger.debug(queuedJobs);
-    const metaDataOps = await metadataTransaction.exec();
-    this.checkTransactionResult(metaDataOps);
-
-    return {
-      assetIds: references,
-    };
-  }
-
   // eslint-disable-next-line class-methods-use-this
   private calculateJobId(jobWithoutId: IRequestJob): string {
     const stringVal = JSON.stringify(jobWithoutId);