如何创建代码管道以从存储在 github 的 java 代码构建 jar 文件并将其部署到 lambda 函数？

Question

我想构建一个代码管道，它将从 github 获取代码（java）构建一个 jar 文件并将其部署到 aws lamda（或将 jar 存储在特定的 S3 存储桶中）。我只想使用AWS平台提供的工具。

如果我只使用 Codebuild，我可以从 github 代码构建 jar 并将其存储到 S3(https://docs.aws.amazon.com/codebuild/latest/userguide/getting-started.html)，并且我正在使用部署程序 lamda 函数将代码部署到我的服务 lamda。每当 S3 存储桶部署程序发生任何变化时，都会触发 lamda。

缺点：这个问题是我必须在每次将更改提交到 github 后手动运行 codebuild。我希望此代码构建能够自动检测来自 github 的更改。

为了解决上述问题，我制作了一个代码管道，它使用 github webhooks 检测代码更改，但这里它创建的是 zip 文件而不是 jar

所以我真正尝试的是：

GitHub(changes)--->codebuild-->将jar文件存储到具有特定名称的特定S3存储桶或部署到lambda

buildspec.yml

    version: 0.2

    phases:
  build:
    commands:
      - echo Build started on `date`
      - mvn test
  post_build:
    commands:
      - echo Build completed on `date`
      - mvn package
artifacts:
  files:
    - target/testfunction-1.0.0-jar-with-dependencies.jar

Answer 1

首先，在设置一个简单的管道以在 GitHub 提交发生时更新 lambda 时，CodeDeploy 令人困惑。不应该这么难。我们创建了以下 Lambda 函数，它可以处理 CodePipeline 作业构建工件 (ZIP) 并使用 updateFunctionCode 将 JAR 更新推送到 Lambda。

import com.amazonaws.services.codepipeline.AWSCodePipeline;
import com.amazonaws.services.codepipeline.AWSCodePipelineClientBuilder;
import com.amazonaws.services.codepipeline.model.FailureDetails;
import com.amazonaws.services.codepipeline.model.PutJobFailureResultRequest;
import com.amazonaws.services.codepipeline.model.PutJobSuccessResultRequest;
import com.amazonaws.services.lambda.AWSLambda;
import com.amazonaws.services.lambda.AWSLambdaClientBuilder;
import com.amazonaws.services.lambda.model.UpdateFunctionCodeRequest;
import com.amazonaws.services.lambda.runtime.Context;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import org.json.JSONObject;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

/**
 * Created by jonathan and josh on 1/22/2019.
 * <p>
 * Process Code Pipeline Job
 */
@SuppressWarnings("unused")
public class CodePipelineLambdaUpdater {
  private static AWSCodePipeline codepipeline = null;
  private static AmazonS3 s3 = null;
  private static AWSLambda lambda = null;

  @SuppressWarnings("UnusedParameters")
  public void handler(InputStream inputStream, OutputStream outputStream, Context context) throws IOException {
    // Read the the job JSON object
    String json = new String(readStreamToByteArray(inputStream), "UTF-8");
    JSONObject eventJsonObject = new JSONObject(json);

    // Extract the jobId first
    JSONObject codePiplineJobJsonObject = eventJsonObject.getJSONObject("CodePipeline.job");
    String jobId = codePiplineJobJsonObject.getString("id");

    // Initialize the code pipeline client if necessary
    if (codepipeline == null) {
      codepipeline = AWSCodePipelineClientBuilder.defaultClient();
    }
    if (s3 == null) {
      s3 = AmazonS3ClientBuilder.defaultClient();
    }
    if (lambda == null) {
      lambda = AWSLambdaClientBuilder.defaultClient();
    }

    try {
      // The bucketName and objectKey refer to the intermediate ZIP file produced by CodePipeline
      String bucketName = codePiplineJobJsonObject.getJSONObject("data").getJSONArray("inputArtifacts").getJSONObject(0).getJSONObject("location").getJSONObject("s3Location").getString("bucketName");
      String objectKey = codePiplineJobJsonObject.getJSONObject("data").getJSONArray("inputArtifacts").getJSONObject(0).getJSONObject("location").getJSONObject("s3Location").getString("objectKey");
      // The user parameter is the Lambda function name that we want to update.  This is configured when adding the CodePipeline Action
      String functionName = codePiplineJobJsonObject.getJSONObject("data").getJSONObject("actionConfiguration").getJSONObject("configuration").getString("UserParameters");

      System.out.println("bucketName: " + bucketName);
      System.out.println("objectKey: " + objectKey);
      System.out.println("functionName: " + functionName);

      // Download the object
      S3Object s3Object = s3.getObject(new GetObjectRequest(bucketName, objectKey));

      // Read the JAR out of the ZIP file.  Should be the only file for our Java code
      ZipInputStream zis = new ZipInputStream(s3Object.getObjectContent());
      ZipEntry zipEntry;
      byte[] data = null;
      //noinspection LoopStatementThatDoesntLoop
      while ((zipEntry = zis.getNextEntry()) != null) {
        if (zipEntry.getName().endsWith(".jar")) {
          System.out.println("zip file: " + zipEntry.getName());
          data = readStreamToByteArray(zis);
          System.out.println("Length: " + data.length);
          break;
        }
      }

      // If we have data then update the function
      if (data != null) {
        // Update the lambda function
        UpdateFunctionCodeRequest updateFunctionCodeRequest = new UpdateFunctionCodeRequest();
        updateFunctionCodeRequest.setFunctionName(functionName);
        updateFunctionCodeRequest.setPublish(true);
        updateFunctionCodeRequest.setZipFile(ByteBuffer.wrap(data));
        lambda.updateFunctionCode(updateFunctionCodeRequest);
        System.out.println("Updated function: " + functionName);

        // Indicate success
        PutJobSuccessResultRequest putJobSuccessResultRequest = new PutJobSuccessResultRequest();
        putJobSuccessResultRequest.setJobId(jobId);
        codepipeline.putJobSuccessResult(putJobSuccessResultRequest);
      } else {
        // Failre the job
        PutJobFailureResultRequest putJobFailureResultRequest = new PutJobFailureResultRequest();
        putJobFailureResultRequest.setJobId(jobId);
        FailureDetails failureDetails = new FailureDetails();
        failureDetails.setMessage("No data available to update function with.");
        putJobFailureResultRequest.setFailureDetails(failureDetails);
        codepipeline.putJobFailureResult(putJobFailureResultRequest);
      }

      System.out.println("Finished");
    } catch (Throwable e) {
      // Handle all other exceptions
      System.out.println("Well that ended badly...");
      e.printStackTrace();

      PutJobFailureResultRequest putJobFailureResultRequest = new PutJobFailureResultRequest();
      putJobFailureResultRequest.setJobId(jobId);
      FailureDetails failureDetails = new FailureDetails();
      failureDetails.setMessage("Failed with error: " + e.getMessage());
      putJobFailureResultRequest.setFailureDetails(failureDetails);
      codepipeline.putJobFailureResult(putJobFailureResultRequest);
    }
  }

  private static void copy(InputStream in, OutputStream out) throws IOException {
    byte[] buffer = new byte[100000];


    for (; ; ) {
      int rc = in.read(buffer);
      if (rc == -1) break;
      out.write(buffer, 0, rc);
    }

    out.flush();
  }

  private static byte[] readStreamToByteArray(InputStream in) throws IOException {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();

    try {
      copy(in, baos);
    } finally {
      safeClose(in);
    }

    return baos.toByteArray();
  }

  private static InputStream safeClose(InputStream in) {
    try {
      if (in != null) in.close();
    } catch (Throwable ignored) {
    }
    return null;
  }
}

这是项目 Maven 文件。

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.yourcompany</groupId>
    <artifactId>codepipeline-lambda-updater</artifactId>
    <version>1.0-SNAPSHOT</version>

    <dependencyManagement>
        <dependencies>
            <dependency>
                <groupId>com.amazonaws</groupId>
                <artifactId>aws-java-sdk-bom</artifactId>
                <version>1.11.487</version>
                <type>pom</type>
                <scope>import</scope>
            </dependency>
        </dependencies>
    </dependencyManagement>

    <dependencies>
        <dependency>
            <groupId>com.amazonaws</groupId>
            <artifactId>aws-lambda-java-core</artifactId>
            <version>1.1.0</version>
        </dependency>
        <dependency>
            <groupId>com.amazonaws</groupId>
            <artifactId>aws-java-sdk-lambda</artifactId>
        </dependency>
        <dependency>
            <groupId>com.amazonaws</groupId>
            <artifactId>aws-java-sdk-core</artifactId>
        </dependency>
        <!-- https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-s3 -->
        <dependency>
            <groupId>com.amazonaws</groupId>
            <artifactId>aws-java-sdk-s3</artifactId>
            <version>1.11.487</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-codepipeline -->
        <dependency>
            <groupId>com.amazonaws</groupId>
            <artifactId>aws-java-sdk-codepipeline</artifactId>
            <version>1.11.487</version>
        </dependency>

        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-api</artifactId>
            <version>2.10.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-core</artifactId>
            <version>2.10.0</version>
        </dependency>
        <dependency>
            <groupId>com.amazonaws</groupId>
            <artifactId>aws-lambda-java-log4j2</artifactId>
            <version>1.0.0</version>
        </dependency>
        <dependency>
            <groupId>org.jetbrains</groupId>
            <artifactId>annotations</artifactId>
            <version>15.0</version>
        </dependency>
        <!--<dependency>-->
            <!--<groupId>com.google.code.gson</groupId>-->
            <!--<artifactId>gson</artifactId>-->
            <!--<version>2.8.2</version>-->
        <!--</dependency>-->
        <!-- https://mvnrepository.com/artifact/org.json/json -->
        <dependency>
            <groupId>org.json</groupId>
            <artifactId>json</artifactId>
            <version>20180813</version>
        </dependency>
        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-lang3</artifactId>
            <version>3.1</version>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                </configuration>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-shade-plugin</artifactId>
                <version>2.4.3</version>
                <executions>
                    <execution>
                        <phase>package</phase>
                        <goals>
                            <goal>shade</goal>
                        </goals>
                        <configuration>
                            <transformers>
                                <transformer
                                        implementation="com.github.edwgiz.mavenShadePlugin.log4j2CacheTransformer.PluginsCacheFileTransformer">
                                </transformer>
                            </transformers>
                        </configuration>
                    </execution>
                </executions>
                <dependencies>
                    <dependency>
                        <groupId>com.github.edwgiz</groupId>
                        <artifactId>maven-shade-plugin.log4j2-cachefile-transformer</artifactId>
                        <version>2.8.1</version>
                    </dependency>
                </dependencies>
            </plugin>
        </plugins>
    </build>

</project>

此基线应该可以帮助您入门。使用您认为合适的更多 SDK 调用来美化代码以进行更高级的部署。

Answer 2

每个管道执行的 CodePipeline 工件位置不同，因此它们是隔离的。

我认为您要做的是在 CodeBuild 中生成一个 JAR 文件，该文件最终会生成一个 ZIP 格式的 CodePipeline 工件。您可以添加第二个 CodeBuild 操作，该操作接受第一个 CodeBuild 操作的输出（CodeBuild 操作将为您解压缩输入工件）并部署到 S3（这对于使用 AWS CLI 编写脚本非常简单）。

完全可以将两个 CodeBuild 操作结合起来，但我喜欢将“构建”和“部署”步骤分开。