-
Notifications
You must be signed in to change notification settings - Fork 64
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add Google Cloud Storage support for bundles
- Loading branch information
1 parent
97b2113
commit b9e035c
Showing
12 changed files
with
801 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,39 @@ | ||
//Copyright 2019-2024 VMware, Inc. | ||
//SPDX-License-Identifier: EPL-2.0 | ||
apply plugin: 'java-library' | ||
apply plugin: 'org.springframework.boot' | ||
apply plugin: 'io.spring.dependency-management' | ||
|
||
archivesBaseName = 'md-data-api-gcsimpl' | ||
|
||
|
||
|
||
jar { | ||
manifest { | ||
attributes 'Implementation-Title': 'md-data-api-gcsimpl', | ||
'Implementation-Version': version | ||
} | ||
} | ||
|
||
dependencies { | ||
|
||
api project(":md-data-api") | ||
api project(":vip-common") | ||
compileOnly("org.springframework.boot:spring-boot") | ||
compileOnly("org.springframework.boot:spring-boot-starter-data-jpa") | ||
compileOnly("org.slf4j:slf4j-api:$slf4jVersion") | ||
compileOnly("org.apache.commons:commons-lang3:$commonsLangVersion") | ||
compileOnly("com.fasterxml.jackson.core:jackson-databind:$jacksonVersion") | ||
implementation ("com.google.cloud:google-cloud-storage:2.42.0") | ||
implementation ("com.google.cloud:google-cloud-storage-control:2.42.0") | ||
|
||
} | ||
|
||
bootJar { | ||
enabled = false | ||
} | ||
|
||
jar { | ||
classifier = '' | ||
enabled = true | ||
} |
68 changes: 68 additions & 0 deletions
68
...simpl/src/main/java/com/vmware/vip/messages/data/dao/gcs/impl/GcsComponentChannelDao.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,68 @@ | ||
/* | ||
* Copyright 2019-2024 VMware, Inc. | ||
* SPDX-License-Identifier: EPL-2.0 | ||
*/ | ||
package com.vmware.vip.messages.data.dao.gcs.impl; | ||
|
||
import java.io.ByteArrayInputStream; | ||
import java.nio.channels.Channels; | ||
import java.util.ArrayList; | ||
import java.util.List; | ||
|
||
import org.slf4j.Logger; | ||
import org.slf4j.LoggerFactory; | ||
import org.springframework.beans.factory.annotation.Autowired; | ||
import org.springframework.context.annotation.Profile; | ||
import org.springframework.stereotype.Repository; | ||
|
||
import com.google.cloud.storage.Blob; | ||
import com.google.cloud.storage.BlobId; | ||
import com.vmware.vip.common.constants.ConstantsChar; | ||
import com.vmware.vip.common.i18n.resourcefile.ResourceFilePathGetter; | ||
import com.vmware.vip.messages.data.dao.api.IComponentChannelDao; | ||
import com.vmware.vip.messages.data.dao.exception.DataException; | ||
import com.vmware.vip.messages.data.dao.model.ResultMessageChannel; | ||
import com.vmware.vip.messages.data.gcs.conf.GcsClient; | ||
import com.vmware.vip.messages.data.gcs.conf.GcsConfig; | ||
import com.vmware.vip.messages.data.gcs.util.GcsUtils; | ||
|
||
@Profile("gcs") | ||
@Repository | ||
public class GcsComponentChannelDao implements IComponentChannelDao { | ||
private static Logger logger = LoggerFactory.getLogger(GcsComponentChannelDao.class); | ||
|
||
@Autowired | ||
private GcsClient gcsClient; | ||
|
||
@Autowired | ||
private GcsConfig config; | ||
|
||
@Override | ||
public List<ResultMessageChannel> getTransReadableByteChannels(String productName, String version, | ||
List<String> components, List<String> locales) throws DataException { | ||
logger.debug("GcsComponentChannelDao.getTransReadableByteChannels()-> product={}, version={}, components={}, locales={}", | ||
productName, version, components, locales); | ||
List<ResultMessageChannel> resultChannels = new ArrayList<ResultMessageChannel>(); | ||
for (String component : components) { | ||
for (String locale : locales) { | ||
String filePath = GcsUtils.genProductVersionGcsPath(productName, version) + component + ConstantsChar.BACKSLASH | ||
+ ResourceFilePathGetter.getLocalizedJSONFileName(locale); | ||
BlobId blobId = BlobId.of(config.getBucketName(), filePath); | ||
Blob blob = gcsClient.getGcsStorage().get(blobId); | ||
if (blob != null) { | ||
byte[] content = gcsClient.getGcsStorage().readAllBytes(config.getBucketName(), filePath); | ||
if (content != null) { | ||
ByteArrayInputStream is = new ByteArrayInputStream(content); | ||
resultChannels.add(new ResultMessageChannel(component, locale, Channels.newChannel(is))); | ||
} | ||
|
||
} | ||
|
||
} | ||
} | ||
logger.debug("fileSize: {}", resultChannels.size()); | ||
|
||
return resultChannels; | ||
} | ||
|
||
} |
82 changes: 82 additions & 0 deletions
82
...impl/src/main/java/com/vmware/vip/messages/data/dao/gcs/impl/GcsMultComponentDaoImpl.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,82 @@ | ||
/** | ||
* Copyright 2019-2024 VMware, Inc. | ||
* SPDX-License-Identifier: EPL-2.0 | ||
*/ | ||
package com.vmware.vip.messages.data.dao.gcs.impl; | ||
|
||
import java.util.ArrayList; | ||
import java.util.List; | ||
import org.slf4j.Logger; | ||
import org.slf4j.LoggerFactory; | ||
import org.springframework.beans.factory.annotation.Autowired; | ||
import org.springframework.context.annotation.Profile; | ||
import org.springframework.stereotype.Repository; | ||
import com.vmware.vip.common.constants.ConstantsChar; | ||
import com.vmware.vip.messages.data.dao.api.IMultComponentDao; | ||
import com.vmware.vip.messages.data.dao.api.IOneComponentDao; | ||
import com.vmware.vip.messages.data.dao.exception.DataException; | ||
import com.vmware.vip.messages.data.dao.model.ResultI18Message; | ||
/** | ||
* this class get the bundle json from bundle file | ||
*/ | ||
@Repository | ||
@Profile("gcs") | ||
public class GcsMultComponentDaoImpl implements IMultComponentDao { | ||
private static Logger logger = LoggerFactory.getLogger(GcsMultComponentDaoImpl.class); | ||
|
||
@Autowired | ||
private IOneComponentDao oneComponentDao; | ||
|
||
/** | ||
* get the bundle files from gcs service | ||
*/ | ||
@Override | ||
public List<String> get2JsonStrs(String productName, String version, List<String> components, | ||
List<String> locales) throws DataException { | ||
logger.debug("begin get2JsonStrs"); | ||
List<String> bundles = new ArrayList<>(); | ||
if (components == null || locales == null) { | ||
throw new DataException("Gcs No component or locale"); | ||
} | ||
for (String component : components) { | ||
for (String locale : locales) { | ||
try { | ||
bundles.add(oneComponentDao.get2JsonStr(productName, version, component, locale)); | ||
}catch(DataException e ) { | ||
logger.warn(e.getMessage(), e); | ||
} | ||
} | ||
} | ||
logger.debug("end get2JsonStrs"); | ||
return bundles; | ||
} | ||
|
||
/** | ||
* get the bundle files and convert to ResultI18Message | ||
*/ | ||
@Override | ||
public List<ResultI18Message> get(String productName, String version, List<String> components, | ||
List<String> locales) throws DataException { | ||
logger.debug("begin get"); | ||
List<ResultI18Message> bundles = new ArrayList<>(); | ||
if (components == null || locales == null) { | ||
throw new DataException("Gcs No component or locale"); | ||
} | ||
for (String component : components) { | ||
for (String locale : locales) { | ||
try { | ||
bundles.add(oneComponentDao.get(productName, version, component, locale)); | ||
} catch (DataException e) { | ||
throw new DataException("Gcs Failed to get for " + productName | ||
+ ConstantsChar.BACKSLASH + version + ConstantsChar.BACKSLASH + component | ||
+ ConstantsChar.BACKSLASH + locale + ".", e); | ||
} | ||
} | ||
} | ||
logger.debug("end get"); | ||
if (bundles.size() == 0) { | ||
throw new DataException("Gcs No bundle is found."); | ||
} | ||
return bundles; | ||
} | ||
} |
142 changes: 142 additions & 0 deletions
142
...simpl/src/main/java/com/vmware/vip/messages/data/dao/gcs/impl/GcsOneComponentDaoImpl.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,142 @@ | ||
/** | ||
* Copyright 2019-2024 VMware, Inc. | ||
* SPDX-License-Identifier: EPL-2.0 | ||
*/ | ||
package com.vmware.vip.messages.data.dao.gcs.impl; | ||
|
||
import java.io.IOException; | ||
import java.nio.charset.StandardCharsets; | ||
import java.util.HashMap; | ||
import java.util.Map; | ||
import org.slf4j.Logger; | ||
import org.slf4j.LoggerFactory; | ||
import org.springframework.beans.factory.annotation.Autowired; | ||
import org.springframework.context.annotation.Profile; | ||
import org.springframework.stereotype.Repository; | ||
import org.springframework.util.StringUtils; | ||
import com.google.cloud.storage.Blob; | ||
import com.google.cloud.storage.BlobId; | ||
import com.google.cloud.storage.BlobInfo; | ||
import com.google.cloud.storage.Storage; | ||
import com.fasterxml.jackson.core.JsonProcessingException; | ||
import com.fasterxml.jackson.databind.ObjectMapper; | ||
import com.vmware.vip.common.constants.ConstantsChar; | ||
import com.vmware.vip.common.constants.ConstantsFile; | ||
import com.vmware.vip.common.constants.ConstantsKeys; | ||
import com.vmware.vip.common.i18n.resourcefile.ResourceFilePathGetter; | ||
import com.vmware.vip.messages.data.dao.api.IOneComponentDao; | ||
import com.vmware.vip.messages.data.dao.exception.DataException; | ||
import com.vmware.vip.messages.data.dao.model.ResultI18Message; | ||
import com.vmware.vip.messages.data.gcs.conf.GcsClient; | ||
import com.vmware.vip.messages.data.gcs.conf.GcsConfig; | ||
import com.vmware.vip.messages.data.gcs.util.GcsUtils; | ||
|
||
/** | ||
* This java class is used to handle translation bundle file or translation | ||
*/ | ||
@Repository | ||
@Profile("gcs") | ||
public class GcsOneComponentDaoImpl implements IOneComponentDao { | ||
|
||
@Autowired | ||
private GcsClient gcsClient; | ||
|
||
@Autowired | ||
private GcsConfig config; | ||
|
||
static final String S3_NOT_EXIST_STR = "S3 File doesn't exist: "; | ||
private static Logger logger = LoggerFactory.getLogger(GcsOneComponentDaoImpl.class); | ||
|
||
/** | ||
* get one component bundle files from s3 server and convert to ResultI18Message | ||
* Object | ||
*/ | ||
@Override | ||
public ResultI18Message get(String productName, String version, String component, String locale) | ||
throws DataException { | ||
String jsonStr = get2JsonStr(productName, version, component, locale); | ||
ObjectMapper mapper = new ObjectMapper(); | ||
ResultI18Message result = null; | ||
try { | ||
result = mapper.readValue(jsonStr, ResultI18Message.class); | ||
} catch (IOException e) { | ||
String errorLog = ConstantsKeys.FATA_ERROR + e.getMessage(); | ||
logger.error(errorLog, e); | ||
throw new DataException(S3_NOT_EXIST_STR); | ||
} | ||
if (result != null) { | ||
result.setProduct(productName); | ||
result.setVersion(version); | ||
result.setComponent(component); | ||
result.setLocale(locale); | ||
} else { | ||
throw new DataException(S3_NOT_EXIST_STR); | ||
} | ||
return result; | ||
} | ||
|
||
/** | ||
* get one component bundle files from s3 server as json String | ||
*/ | ||
@Override | ||
public String get2JsonStr(String productName, String version, String component, String locale) | ||
throws DataException { | ||
logger.debug("GcsOneComponentDaoImpl.get2JsonStr()"); | ||
String filePath = GcsUtils.genProductVersionGcsPath(productName, version) + component + ConstantsChar.BACKSLASH | ||
+ ResourceFilePathGetter.getLocalizedJSONFileName(locale); | ||
BlobId blobId = BlobId.of(config.getBucketName(), filePath); | ||
Blob blob = gcsClient.getGcsStorage().get(blobId); | ||
if (blob != null) { | ||
byte[] content = blob.getContent(); | ||
if (content != null) { | ||
return new String(content, StandardCharsets.UTF_8); | ||
} | ||
} | ||
|
||
throw new DataException(S3_NOT_EXIST_STR + filePath); | ||
} | ||
|
||
@Override | ||
public boolean add(String productName, String version, String component, String locale, | ||
Map<String, String> messages) throws DataException { | ||
return false; | ||
} | ||
|
||
/** | ||
* update the component bundle file to remote S3 server | ||
*/ | ||
@Override | ||
public boolean update(String productName, String version, String component, String locale, | ||
Map<String, String> messages) throws DataException { | ||
logger.debug("GcsOneComponentDaoImpl.update()-> productName={}, component={}, version={}, locale={}, messages={}", | ||
productName, component,version, locale, messages); | ||
if (StringUtils.isEmpty(component)) { | ||
component = ConstantsFile.DEFAULT_COMPONENT; | ||
} | ||
String filePath = GcsUtils.genProductVersionGcsPath(productName, version) + component + ConstantsChar.BACKSLASH | ||
+ ResourceFilePathGetter.getLocalizedJSONFileName(locale); | ||
Map<String, Object> json = new HashMap<String, Object>(); | ||
json.put(ConstantsKeys.COMPONENT, component); | ||
json.put(ConstantsKeys.lOCALE, locale); | ||
json.put(ConstantsKeys.MESSAGES, messages); | ||
String content; | ||
try { | ||
content = new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(json); | ||
} catch (JsonProcessingException e) { | ||
throw new DataException(ConstantsKeys.FATA_ERROR + "Failed to convert content to file: " + filePath + ".", | ||
e); | ||
} | ||
BlobId blobId = BlobId.of(config.getBucketName(), filePath); | ||
BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build(); | ||
Storage.BlobTargetOption precondition = Storage.BlobTargetOption | ||
.generationMatch(gcsClient.getGcsStorage().get(config.getBucketName(), filePath).getGeneration()); | ||
gcsClient.getGcsStorage().create(blobInfo, content.getBytes(StandardCharsets.UTF_8), precondition); | ||
return true; | ||
} | ||
|
||
@Override | ||
public boolean delete(String productName, String version, String component, String locale) throws DataException { | ||
return false; | ||
} | ||
|
||
} |
Oops, something went wrong.