Skip to content

Commit fbd68e6

Browse files
miniscruffJordi Bertran de Balanda
miniscruff
authored and
Jordi Bertran de Balanda
committed
feature: CLDSRV-162 use metadata and data from new arsenal
1 parent e109daa commit fbd68e6

File tree

127 files changed

+1296
-6092
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

127 files changed

+1296
-6092
lines changed

‎Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ RUN cd /tmp \
3333
&& rm -rf /tmp/Python-$PY_VERSION.tgz
3434

3535
RUN yarn cache clean \
36-
&& yarn install --production --ignore-optional --ignore-engines --network-concurrency 1 \
36+
&& yarn install --production --ignore-optional --ignore-engines --network-concurrency 1 \
3737
&& apt-get autoremove --purge -y python git build-essential \
3838
&& rm -rf /var/lib/apt/lists/* \
3939
&& yarn cache clean \

‎DockerfileMem

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ COPY . /usr/src/app
77

88
RUN apt-get update \
99
&& apt-get install -y jq python git build-essential --no-install-recommends \
10-
&& yarn install --production --network-concurrency 1 \
10+
&& yarn install --production \
1111
&& apt-get autoremove --purge -y python git build-essential \
1212
&& rm -rf /var/lib/apt/lists/* \
1313
&& yarn cache clean \

‎constants.js

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -174,10 +174,6 @@ const constants = {
174174
'bucket',
175175
],
176176
allowedUtapiEventFilterStates: ['allow', 'deny'],
177-
// The AWS assumed Role resource type
178-
assumedRoleArnResourceType: 'assumed-role',
179-
// Session name of the backbeat lifecycle assumed role session.
180-
backbeatLifecycleSessionName: 'backbeat-lifecycle',
181177
};
182178

183179
module.exports = constants;

‎lib/Config.js

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,60 @@ const versionIdUtils = versioning.VersionID;
2424
const defaultHealthChecks = { allowFrom: ['127.0.0.1/8', '::1'] };
2525

2626
const defaultLocalCache = { host: '127.0.0.1', port: 6379 };
27+
const defaultExternalBackendsConfig = {
28+
// eslint-disable-next-line camelcase
29+
aws_s3: {
30+
httpAgent: {
31+
keepAlive: false,
32+
keepAliveMsecs: 1000,
33+
maxFreeSockets: 256,
34+
maxSockets: null,
35+
},
36+
},
37+
gcp: {
38+
httpAgent: {
39+
keepAlive: true,
40+
keepAliveMsecs: 1000,
41+
maxFreeSockets: 256,
42+
maxSockets: null,
43+
},
44+
},
45+
};
2746

2847
const gcpScope = 'https://www.googleapis.com/auth/cloud-platform';
2948

49+
function assertCertPaths(key, cert, ca, basePath) {
50+
const certObj = {};
51+
certObj.paths = {};
52+
certObj.certs = {};
53+
if (key) {
54+
const keypath = key.startsWith('/') ? key : `${basePath}/${key}`;
55+
assert.doesNotThrow(() =>
56+
fs.accessSync(keypath, fs.F_OK | fs.R_OK),
57+
`File not found or unreachable: ${keypath}`);
58+
certObj.paths.key = keypath;
59+
certObj.certs.key = fs.readFileSync(keypath, 'ascii');
60+
}
61+
if (cert) {
62+
const certpath = cert.startsWith('/') ? cert : `${basePath}/${cert}`;
63+
assert.doesNotThrow(() =>
64+
fs.accessSync(certpath, fs.F_OK | fs.R_OK),
65+
`File not found or unreachable: ${certpath}`);
66+
certObj.paths.cert = certpath;
67+
certObj.certs.cert = fs.readFileSync(certpath, 'ascii');
68+
}
69+
70+
if (ca) {
71+
const capath = ca.startsWith('/') ? ca : `${basePath}/${ca}`;
72+
assert.doesNotThrow(() =>
73+
fs.accessSync(capath, fs.F_OK | fs.R_OK),
74+
`File not found or unreachable: ${capath}`);
75+
certObj.paths.ca = capath;
76+
certObj.certs.ca = fs.readFileSync(capath, 'ascii');
77+
}
78+
return certObj;
79+
}
80+
3081
function parseSproxydConfig(configSproxyd) {
3182
const joiSchema = joi.object({
3283
bootstrap: joi.array().items(joi.string()).min(1),
@@ -1124,6 +1175,58 @@ class Config extends EventEmitter {
11241175
'certFilePaths.cert must be defined');
11251176
}
11261177

1178+
this.outboundProxy = {};
1179+
const envProxy = process.env.HTTP_PROXY || process.env.HTTPS_PROXY
1180+
|| process.env.http_proxy || process.env.https_proxy;
1181+
const p = config.outboundProxy;
1182+
const proxyUrl = envProxy || (p ? p.url : '');
1183+
if (proxyUrl) {
1184+
assert(typeof proxyUrl === 'string',
1185+
'bad proxy config: url must be a string');
1186+
const { protocol, hostname, port, auth } = url.parse(proxyUrl);
1187+
assert(protocol === 'http:' || protocol === 'https:',
1188+
'bad proxy config: protocol must be http or https');
1189+
assert(typeof hostname === 'string' && hostname !== '',
1190+
'bad proxy config: hostname must be a non-empty string');
1191+
if (port) {
1192+
const portInt = Number.parseInt(port, 10);
1193+
assert(!Number.isNaN(portInt) && portInt > 0,
1194+
'bad proxy config: port must be a number greater than 0');
1195+
}
1196+
if (auth) {
1197+
assert(typeof auth === 'string',
1198+
'bad proxy config: auth must be string');
1199+
const authArray = auth.split(':');
1200+
assert(authArray.length === 2 && authArray[0].length > 0
1201+
&& authArray[1].length > 0, 'bad proxy config: ' +
1202+
'auth must be of format username:password');
1203+
}
1204+
this.outboundProxy.url = proxyUrl;
1205+
this.outboundProxy.certs = {};
1206+
const envCert = process.env.HTTPS_PROXY_CERTIFICATE;
1207+
const key = p ? p.key : '';
1208+
const cert = p ? p.cert : '';
1209+
const caBundle = envCert || (p ? p.caBundle : '');
1210+
if (p) {
1211+
assert(typeof p === 'object',
1212+
'bad config: "proxy" should be an object');
1213+
}
1214+
if (key) {
1215+
assert(typeof key === 'string',
1216+
'bad config: proxy.key should be a string');
1217+
}
1218+
if (cert) {
1219+
assert(typeof cert === 'string',
1220+
'bad config: proxy.cert should be a string');
1221+
}
1222+
if (caBundle) {
1223+
assert(typeof caBundle === 'string',
1224+
'bad config: proxy.caBundle should be a string');
1225+
}
1226+
const certObj = assertCertPaths(key, cert, caBundle, this._basePath);
1227+
this.outboundProxy.certs = certObj.certs;
1228+
}
1229+
11271230
// Ephemeral token to protect the reporting endpoint:
11281231
// try inherited from parent first, then hardcoded in conf file,
11291232
// then create a fresh one as last resort.
@@ -1132,6 +1235,40 @@ class Config extends EventEmitter {
11321235
config.reportToken ||
11331236
uuidv4();
11341237

1238+
// External backends
1239+
// Currently supports configuring httpAgent(s) for keepAlive
1240+
this.externalBackends = defaultExternalBackendsConfig;
1241+
if (config.externalBackends) {
1242+
const extBackendsConfig = Object.keys(config.externalBackends);
1243+
extBackendsConfig.forEach(b => {
1244+
// assert that it's a valid backend
1245+
assert(externalBackends[b] !== undefined,
1246+
`bad config: ${b} is not one of valid external backends: ` +
1247+
`${Object.keys(externalBackends).join(', ')}`);
1248+
1249+
const { httpAgent } = config.externalBackends[b];
1250+
assert(typeof httpAgent === 'object',
1251+
`bad config: ${b} must have httpAgent object defined`);
1252+
const { keepAlive, keepAliveMsecs, maxFreeSockets, maxSockets }
1253+
= httpAgent;
1254+
assert(typeof keepAlive === 'boolean',
1255+
`bad config: ${b}.httpAgent.keepAlive must be a boolean`);
1256+
assert(typeof keepAliveMsecs === 'number' &&
1257+
httpAgent.keepAliveMsecs > 0,
1258+
`bad config: ${b}.httpAgent.keepAliveMsecs must be` +
1259+
' a number > 0');
1260+
assert(typeof maxFreeSockets === 'number' &&
1261+
httpAgent.maxFreeSockets >= 0,
1262+
`bad config: ${b}.httpAgent.maxFreeSockets must be ` +
1263+
'a number >= 0');
1264+
assert((typeof maxSockets === 'number' && maxSockets >= 0) ||
1265+
maxSockets === null,
1266+
`bad config: ${b}.httpAgent.maxFreeSockets must be ` +
1267+
'null or a number >= 0');
1268+
Object.assign(this.externalBackends[b].httpAgent, httpAgent);
1269+
});
1270+
}
1271+
11351272
// requests-proxy configuration
11361273
this.requests = {
11371274
viaProxy: false,

‎lib/api/apiUtils/authorization/permissionChecks.js

Lines changed: 1 addition & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
const { evaluators, actionMaps, RequestContext } = require('arsenal').policies;
22
const constants = require('../../../../constants');
33

4-
const { allAuthedUsersId, bucketOwnerActions, logId, publicId,
5-
assumedRoleArnResourceType, backbeatLifecycleSessionName } = constants;
4+
const { allAuthedUsersId, bucketOwnerActions, logId, publicId } = constants;
65

76
// whitelist buckets to allow public read on objects
87
const publicReadBuckets = process.env.ALLOW_PUBLIC_READ_BUCKETS ?
@@ -365,34 +364,10 @@ function validatePolicyResource(bucketName, policy) {
365364
});
366365
}
367366

368-
/** isLifecycleSession - check if it is the Lifecycle assumed role session arn.
369-
* @param {string} arn - Amazon resource name - example:
370-
* arn:aws:sts::257038443293:assumed-role/rolename/backbeat-lifecycle
371-
* @return {boolean} true if Lifecycle assumed role session arn, false if not.
372-
*/
373-
function isLifecycleSession(arn) {
374-
if (!arn) {
375-
return false;
376-
}
377-
378-
const arnSplits = arn.split(':');
379-
const service = arnSplits[2];
380-
381-
const resourceNames = arnSplits[arnSplits.length - 1].split('/');
382-
383-
const resourceType = resourceNames[0];
384-
const sessionName = resourceNames[resourceNames.length - 1];
385-
386-
return (service === 'sts' &&
387-
resourceType === assumedRoleArnResourceType &&
388-
sessionName === backbeatLifecycleSessionName);
389-
}
390-
391367
module.exports = {
392368
isBucketAuthorized,
393369
isObjAuthorized,
394370
checkBucketAcls,
395371
checkObjectAcls,
396372
validatePolicyResource,
397-
isLifecycleSession,
398373
};

‎lib/api/apiUtils/authorization/tagConditionKeys.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ function updateRequestContexts(request, requestContexts, apiMethod, log, cb) {
4949
return metadata.getObjectMD(bucketName, objectKey, { versionId: reqVersionId }, log,
5050
(err, objMD) => {
5151
if (err) {
52-
if (err.NoSuchKey) {
52+
if (err.is.NoSuchKey) {
5353
return next();
5454
}
5555
log.trace('error getting request object tags');

‎lib/api/apiUtils/bucket/bucketCreation.js

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ function addToUsersBucket(canonicalID, bucketName, log, cb) {
2222

2323
// Get new format usersBucket to see if it exists
2424
return metadata.getBucket(usersBucket, log, (err, usersBucketAttrs) => {
25-
if (err && !err.NoSuchBucket && !err.BucketAlreadyExists) {
25+
if (err && !err.is.NoSuchBucket && !err.is.BucketAlreadyExists) {
2626
return cb(err);
2727
}
2828
const splitter = usersBucketAttrs ?
@@ -36,7 +36,7 @@ function addToUsersBucket(canonicalID, bucketName, log, cb) {
3636
usersBucket : oldUsersBucket;
3737
return metadata.putObjectMD(usersBucketBeingCalled, key,
3838
omVal, {}, log, err => {
39-
if (err && err.NoSuchBucket) {
39+
if (err && err.is.NoSuchBucket) {
4040
// There must be no usersBucket so createBucket
4141
// one using the new format
4242
log.trace('users bucket does not exist, ' +
@@ -56,9 +56,7 @@ function addToUsersBucket(canonicalID, bucketName, log, cb) {
5656
// from getting a BucketAlreadyExists
5757
// error with respect
5858
// to the usersBucket.
59-
if (err &&
60-
err !==
61-
errors.BucketAlreadyExists) {
59+
if (err && !err.is.BucketAlreadyExists) {
6260
log.error('error from metadata', {
6361
error: err,
6462
});
@@ -206,7 +204,7 @@ function createBucket(authInfo, bucketName, headers,
206204
},
207205
getAnyExistingBucketInfo: function getAnyExistingBucketInfo(callback) {
208206
metadata.getBucket(bucketName, log, (err, data) => {
209-
if (err && err.NoSuchBucket) {
207+
if (err && err.is.NoSuchBucket) {
210208
return callback(null, 'NoBucketYet');
211209
}
212210
if (err) {

‎lib/api/apiUtils/bucket/bucketDeletion.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ function _deleteMPUbucket(destinationBucketName, log, cb) {
1616
`${mpuBucketPrefix}${destinationBucketName}`;
1717
return metadata.deleteBucket(mpuBucketName, log, err => {
1818
// If the mpu bucket does not exist, just move on
19-
if (err && err.NoSuchBucket) {
19+
if (err && err.is.NoSuchBucket) {
2020
return cb();
2121
}
2222
return cb(err);
@@ -90,7 +90,7 @@ function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, log, cb) {
9090
log, (err, objectsListRes) => {
9191
// If no shadow bucket ever created, no ongoing MPU's, so
9292
// continue with deletion
93-
if (err && err.NoSuchBucket) {
93+
if (err && err.is.NoSuchBucket) {
9494
return next();
9595
}
9696
if (err) {

‎lib/api/apiUtils/bucket/deleteUserBucketEntry.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ function deleteUserBucketEntry(bucketName, canonicalID, log, cb) {
1111
metadata.deleteObjectMD(usersBucket, keyForUserBucket, {}, log, error => {
1212
// If the object representing the bucket is not in the
1313
// users bucket just continue
14-
if (error && error.NoSuchKey) {
14+
if (error && error.is.NoSuchKey) {
1515
return cb(null);
1616
// BACKWARDS COMPATIBILITY: Remove this once no longer
1717
// have old user bucket format
@@ -20,7 +20,7 @@ function deleteUserBucketEntry(bucketName, canonicalID, log, cb) {
2020
oldSplitter, bucketName);
2121
return metadata.deleteObjectMD(oldUsersBucket, keyForUserBucket2,
2222
{}, log, error => {
23-
if (error && !error.NoSuchKey) {
23+
if (error && !error.is.NoSuchKey) {
2424
log.error('from metadata while deleting user bucket',
2525
{ error });
2626
return cb(error);

‎lib/api/apiUtils/object/abortMultipartUpload.js

Lines changed: 12 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,10 @@
11
const async = require('async');
22

3-
const { config } = require('../../../Config');
43
const constants = require('../../../../constants');
5-
const data = require('../../../data/wrapper');
4+
const { data } = require('../../../data/wrapper');
65
const locationConstraintCheck = require('../object/locationConstraintCheck');
76
const { metadataValidateBucketAndObj } =
87
require('../../../metadata/metadataUtils');
9-
const multipleBackendGateway = require('../../../data/multipleBackendGateway');
108
const services = require('../../../services');
119

1210
function abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
@@ -55,35 +53,19 @@ function abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
5553
return next(err, mpuBucket, mpuOverviewObj, destBucket);
5654
});
5755
},
58-
function ifMultipleBackend(mpuBucket, mpuOverviewObj, destBucket,
56+
function abortExternalMpu(mpuBucket, mpuOverviewObj, destBucket,
5957
next) {
60-
if (config.backends.data === 'multiple') {
61-
let location;
62-
// if controlling location constraint is not stored in object
63-
// metadata, mpu was initiated in legacy S3C, so need to
64-
// determine correct location constraint
65-
if (!mpuOverviewObj.controllingLocationConstraint) {
66-
const backendInfoObj = locationConstraintCheck(request,
67-
null, destBucket, log);
68-
if (backendInfoObj.err) {
69-
return process.nextTick(() => {
70-
next(backendInfoObj.err, destBucket);
71-
});
72-
}
73-
location = backendInfoObj.controllingLC;
74-
} else {
75-
location = mpuOverviewObj.controllingLocationConstraint;
58+
const location = mpuOverviewObj.controllingLocationConstraint;
59+
return data.abortMPU(objectKey, uploadId, location, bucketName,
60+
request, destBucket, locationConstraintCheck, log,
61+
(err, skipDataDelete) => {
62+
if (err) {
63+
return next(err, destBucket);
7664
}
77-
return multipleBackendGateway.abortMPU(objectKey, uploadId,
78-
location, bucketName, log, (err, skipDataDelete) => {
79-
if (err) {
80-
return next(err, destBucket);
81-
}
82-
return next(null, mpuBucket, destBucket,
83-
skipDataDelete);
84-
});
85-
}
86-
return next(null, mpuBucket, destBucket, false);
65+
// for Azure and GCP we do not need to delete data
66+
// for all other backends, skipDataDelete will be set to false
67+
return next(null, mpuBucket, destBucket, skipDataDelete);
68+
});
8769
},
8870
function sendAbortPut(mpuBucket, destBucket, skipDataDelete, next) {
8971
services.sendAbortMPUPut(bucketName, objectKey, uploadId, log,

0 commit comments

Comments
 (0)