Add Google Cloud Storage support for bundles

g11n-ws/build.gradle

@@ -74,6 +74,7 @@ subprojects{
     	postgresqlVersion = "42.1.4"
     	druidVersion = "1.1.8"
     	awsS3Version = "1.12.741"
+        gcpGcsVersion = "26.50.0"
         swaggerVersion = "3.0.0"
         icu4jVersion = "60.3"
 

g11n-ws/modules/md-data-api-gcsimpl/build.gradle

@@ -0,0 +1,39 @@
+//Copyright 2019-2024 VMware, Inc.
+//SPDX-License-Identifier: EPL-2.0
+apply plugin: 'java-library'
+apply plugin: 'org.springframework.boot'
+apply plugin: 'io.spring.dependency-management'
+
+archivesBaseName = 'md-data-api-gcsimpl'
+
+
+
+jar {
+    manifest {
+        attributes 'Implementation-Title': 'md-data-api-gcsimpl',
+                   'Implementation-Version': version
+    }
+}
+
+dependencies {
+
+    api project(":md-data-api")
+    api project(":vip-common")
+    compileOnly("org.springframework.boot:spring-boot")
+    compileOnly("org.springframework.boot:spring-boot-starter-data-jpa")
+    compileOnly("org.slf4j:slf4j-api:$slf4jVersion")
+    compileOnly("org.apache.commons:commons-lang3:$commonsLangVersion")
+    compileOnly("com.fasterxml.jackson.core:jackson-databind:$jacksonVersion")
+    implementation platform("com.google.cloud:libraries-bom:$gcpGcsVersion")
+    implementation("com.google.cloud:google-cloud-storage")
+
+}
+
+bootJar {
+    enabled = false
+}
+
+jar {
+    classifier = ''
+	enabled = true
+}

g11n-ws/modules/md-data-api-gcsimpl/src/main/java/com/vmware/vip/messages/data/dao/gcs/impl/GcsComponentChannelDao.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright 2019-2024 VMware, Inc.
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package com.vmware.vip.messages.data.dao.gcs.impl;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.channels.Channels;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Profile;
+import org.springframework.stereotype.Repository;
+
+import com.google.cloud.ReadChannel;
+import com.google.cloud.storage.Blob;
+import com.google.cloud.storage.BlobId;
+import com.vmware.vip.common.constants.ConstantsChar;
+import com.vmware.vip.common.i18n.resourcefile.ResourceFilePathGetter;
+import com.vmware.vip.messages.data.dao.api.IComponentChannelDao;
+import com.vmware.vip.messages.data.dao.exception.DataException;
+import com.vmware.vip.messages.data.dao.model.ResultMessageChannel;
+import com.vmware.vip.messages.data.gcs.conf.GcsClient;
+import com.vmware.vip.messages.data.gcs.conf.GcsConfig;
+import com.vmware.vip.messages.data.gcs.util.GcsUtils;
+
+@Profile("gcs")
+@Repository
+public class GcsComponentChannelDao implements IComponentChannelDao {
+    private static Logger logger = LoggerFactory.getLogger(GcsComponentChannelDao.class);
+
+    @Autowired
+    private GcsClient gcsClient;
+
+    @Autowired
+    private GcsConfig config;
+
+    @Override
+    public List<ResultMessageChannel> getTransReadableByteChannels(String productName, String version,
+            List<String> components, List<String> locales) throws DataException {
+        logger.debug("GcsComponentChannelDao.getTransReadableByteChannels()-> product={}, version={}, components={}, locales={}",
+                productName, version, components, locales);
+        List<ResultMessageChannel> resultChannels = new ArrayList<ResultMessageChannel>();
+        for (String component : components) {
+            for (String locale : locales) {
+                String filePath = GcsUtils.genProductVersionGcsPath(productName, version) + component + ConstantsChar.BACKSLASH
+                        + ResourceFilePathGetter.getLocalizedJSONFileName(locale);
+                BlobId blobId = BlobId.of(config.getBucketName(), filePath);
+                Blob blob = gcsClient.getGcsStorage().get(blobId);
+                if (blob != null) {
+                	ReadChannel readChannel = blob.reader();
+                	InputStream is = Channels.newInputStream(readChannel);
+                    resultChannels.add(new ResultMessageChannel(component, locale, Channels.newChannel(is)));           
+                }
+            }
+        }
+        logger.debug("fileSize: {}", resultChannels.size());
+
+        return resultChannels;
+    }
+
+}

g11n-ws/modules/md-data-api-gcsimpl/src/main/java/com/vmware/vip/messages/data/dao/gcs/impl/GcsMultComponentDaoImpl.java

@@ -0,0 +1,82 @@
+/**
+ * Copyright 2019-2024 VMware, Inc.
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package com.vmware.vip.messages.data.dao.gcs.impl;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Profile;
+import org.springframework.stereotype.Repository;
+import com.vmware.vip.common.constants.ConstantsChar;
+import com.vmware.vip.messages.data.dao.api.IMultComponentDao;
+import com.vmware.vip.messages.data.dao.api.IOneComponentDao;
+import com.vmware.vip.messages.data.dao.exception.DataException;
+import com.vmware.vip.messages.data.dao.model.ResultI18Message;
+
+/**
+ * this class get the bundle json from bundle file
+ */
+@Repository
+@Profile("gcs")
+public class GcsMultComponentDaoImpl implements IMultComponentDao {
+    private static Logger logger = LoggerFactory.getLogger(GcsMultComponentDaoImpl.class);
+
+    @Autowired
+    private IOneComponentDao oneComponentDao;
+
+    /**
+     * get the bundle files from gcs service
+     */
+    @Override
+    public List<String> get2JsonStrs(String productName, String version, List<String> components, List<String> locales)
+            throws DataException {
+        logger.debug("begin get2JsonStrs");
+        List<String> bundles = new ArrayList<>();
+        if (components == null || locales == null) {
+            throw new DataException("Gcs No component or locale");
+        }
+        for (String component : components) {
+            for (String locale : locales) {
+                try {
+                    bundles.add(oneComponentDao.get2JsonStr(productName, version, component, locale));
+                } catch (DataException e) {
+                    logger.warn(e.getMessage(), e);
+                }
+            }
+        }
+        logger.debug("end get2JsonStrs");
+        return bundles;
+    }
+
+    /**
+     * get the bundle files and convert to ResultI18Message
+     */
+    @Override
+    public List<ResultI18Message> get(String productName, String version, List<String> components, List<String> locales)
+            throws DataException {
+        logger.debug("begin get");
+        List<ResultI18Message> bundles = new ArrayList<>();
+        if (components == null || locales == null) {
+            throw new DataException("Gcs No component or locale");
+        }
+        for (String component : components) {
+            for (String locale : locales) {
+                try {
+                    bundles.add(oneComponentDao.get(productName, version, component, locale));
+                } catch (DataException e) {
+                    throw new DataException("Gcs Failed to get for " + productName + ConstantsChar.BACKSLASH + version
+                            + ConstantsChar.BACKSLASH + component + ConstantsChar.BACKSLASH + locale + ".", e);
+                }
+            }
+        }
+        logger.debug("end get");
+        if (bundles.size() == 0) {
+            throw new DataException("Gcs No bundle is found.");
+        }
+        return bundles;
+    }
+}

g11n-ws/modules/md-data-api-gcsimpl/src/main/java/com/vmware/vip/messages/data/dao/gcs/impl/GcsOneComponentDaoImpl.java

@@ -0,0 +1,143 @@
+/**
+ * Copyright 2019-2024 VMware, Inc.
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package com.vmware.vip.messages.data.dao.gcs.impl;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Profile;
+import org.springframework.stereotype.Repository;
+import org.springframework.util.StringUtils;
+import com.google.cloud.storage.Blob;
+import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.BlobInfo;
+import com.google.cloud.storage.Storage;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.vmware.vip.common.constants.ConstantsChar;
+import com.vmware.vip.common.constants.ConstantsFile;
+import com.vmware.vip.common.constants.ConstantsKeys;
+import com.vmware.vip.common.i18n.resourcefile.ResourceFilePathGetter;
+import com.vmware.vip.messages.data.dao.api.IOneComponentDao;
+import com.vmware.vip.messages.data.dao.exception.DataException;
+import com.vmware.vip.messages.data.dao.model.ResultI18Message;
+import com.vmware.vip.messages.data.gcs.conf.GcsClient;
+import com.vmware.vip.messages.data.gcs.conf.GcsConfig;
+import com.vmware.vip.messages.data.gcs.util.GcsUtils;
+
+/**
+ * This java class is used to handle translation bundle file or translation
+ */
+@Repository
+@Profile("gcs")
+public class GcsOneComponentDaoImpl implements IOneComponentDao {
+
+    @Autowired
+    private GcsClient gcsClient;
+
+    @Autowired
+    private GcsConfig config;
+
+    static final String GCS_NOT_EXIST_STR = "GC Storage File doesn't exist: ";
+    private static Logger logger = LoggerFactory.getLogger(GcsOneComponentDaoImpl.class);
+
+    /**
+     * get one component bundle files from s3 server and convert to ResultI18Message
+     * Object
+     */
+    @Override
+    public ResultI18Message get(String productName, String version, String component, String locale)
+            throws DataException {
+        String jsonStr = get2JsonStr(productName, version, component, locale);
+        ObjectMapper mapper = new ObjectMapper();
+        ResultI18Message result = null;
+        try {
+            result = mapper.readValue(jsonStr, ResultI18Message.class);
+        } catch (IOException e) {
+            String errorLog = ConstantsKeys.FATA_ERROR + e.getMessage();
+            logger.error(errorLog, e);
+            throw new DataException(GCS_NOT_EXIST_STR);
+        }
+        if (result != null) {
+            result.setProduct(productName);
+            result.setVersion(version);
+            result.setComponent(component);
+            result.setLocale(locale);
+        } else {
+            throw new DataException(GCS_NOT_EXIST_STR);
+        }
+        return result;
+    }
+
+    /**
+     * get one component bundle files from s3 server as json String
+     */
+    @Override
+    public String get2JsonStr(String productName, String version, String component, String locale)
+            throws DataException {
+        logger.debug("GcsOneComponentDaoImpl.get2JsonStr()");
+        String filePath = GcsUtils.genProductVersionGcsPath(productName, version) + component + ConstantsChar.BACKSLASH
+                + ResourceFilePathGetter.getLocalizedJSONFileName(locale);
+        BlobId blobId = BlobId.of(config.getBucketName(), filePath);
+        Blob blob = gcsClient.getGcsStorage().get(blobId);
+        if (blob != null) {
+            byte[] content = blob.getContent();
+            if (content != null) {
+                return new String(content, StandardCharsets.UTF_8);
+            }
+        }
+
+        throw new DataException(GCS_NOT_EXIST_STR + filePath);
+    }
+
+    @Override
+    public boolean add(String productName, String version, String component, String locale,
+            Map<String, String> messages) throws DataException {
+        return false;
+    }
+
+    /**
+     * update the component bundle file to remote S3 server
+     */
+    @Override
+    public boolean update(String productName, String version, String componentParam, String locale,
+            Map<String, String> messages) throws DataException {
+        logger.debug("GcsOneComponentDaoImpl.update()-> productName={}, component={}, version={}, locale={}, messages={}",
+                productName, componentParam, version, locale, messages);
+        String component = componentParam;
+        if (StringUtils.isEmpty(component)) {
+        	component = ConstantsFile.DEFAULT_COMPONENT;
+        }
+        String filePath = GcsUtils.genProductVersionGcsPath(productName, version) + component + ConstantsChar.BACKSLASH
+                + ResourceFilePathGetter.getLocalizedJSONFileName(locale);
+        Map<String, Object> json = new HashMap<String, Object>();
+        json.put(ConstantsKeys.COMPONENT, component);
+        json.put(ConstantsKeys.lOCALE, locale);
+        json.put(ConstantsKeys.MESSAGES, messages);
+        String content;
+        try {
+            content = new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(json);
+        } catch (JsonProcessingException e) {
+            throw new DataException(ConstantsKeys.FATA_ERROR + "Failed to convert content to file: " + filePath + ".",
+                    e);
+        }
+        BlobId blobId = BlobId.of(config.getBucketName(), filePath);
+        BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build();
+        Storage.BlobTargetOption precondition = Storage.BlobTargetOption
+                .generationMatch(gcsClient.getGcsStorage().get(config.getBucketName(), filePath).getGeneration());
+        gcsClient.getGcsStorage().create(blobInfo, content.getBytes(StandardCharsets.UTF_8), precondition);
+        return true;
+    }
+
+    @Override
+    public boolean delete(String productName, String version, String component, String locale) throws DataException {
+        return false;
+    }
+
+}