From commits-return-65303-archive-asf-public=cust-asf.ponee.io@camel.apache.org Thu Sep 20 23:05:23 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 4C68C180679 for ; Thu, 20 Sep 2018 23:05:22 +0200 (CEST) Received: (qmail 53160 invoked by uid 500); 20 Sep 2018 21:05:21 -0000 Mailing-List: contact commits-help@camel.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@camel.apache.org Delivered-To: mailing list commits@camel.apache.org Received: (qmail 53116 invoked by uid 99); 20 Sep 2018 21:05:21 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 20 Sep 2018 21:05:21 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id C00CE82F78; Thu, 20 Sep 2018 21:05:20 +0000 (UTC) Date: Thu, 20 Sep 2018 21:05:22 +0000 To: "commits@camel.apache.org" Subject: [camel-k] 02/07: maven: use maven like structure to store artifacts at runtime and pre computed classpath file MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit From: nferraro@apache.org In-Reply-To: <153747752060.20916.4891772111531993222@gitbox.apache.org> References: <153747752060.20916.4891772111531993222@gitbox.apache.org> X-Git-Host: gitbox.apache.org X-Git-Repo: camel-k X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Rev: 6234cca5ea022890a142013791ef62ff30e18d36 X-Git-NotificationType: diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated Message-Id: <20180920210520.C00CE82F78@gitbox.apache.org> This is an automated email from the ASF dual-hosted git repository. nferraro pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/camel-k.git commit 6234cca5ea022890a142013791ef62ff30e18d36 Author: lburgazzoli AuthorDate: Thu Sep 20 01:37:17 2018 +0200 maven: use maven like structure to store artifacts at runtime and pre computed classpath file --- deploy/platform-integration-context-core.yaml | 7 +- deploy/resources.go | 5 +- pkg/apis/camel/v1alpha1/types.go | 1 + pkg/apis/camel/v1alpha1/zz_generated.deepcopy.go | 5 + pkg/build/build_manager.go | 6 +- pkg/build/build_types.go | 12 +-- pkg/build/local/local_builder.go | 72 +++++++------- pkg/build/local/local_builder_test.go | 10 +- pkg/discover/dependencies_test.go | 13 +-- pkg/discover/languages_test.go | 7 +- pkg/stub/action/context/build.go | 1 + pkg/stub/action/integration/deploy.go | 18 ++++ pkg/util/maven/maven.go | 121 ++++++++++++++++------- pkg/util/maven/{types.go => maven_project.go} | 0 14 files changed, 185 insertions(+), 93 deletions(-) diff --git a/deploy/platform-integration-context-core.yaml b/deploy/platform-integration-context-core.yaml index 836d195..3fe794c 100644 --- a/deploy/platform-integration-context-core.yaml +++ b/deploy/platform-integration-context-core.yaml @@ -1,9 +1,12 @@ apiVersion: camel.apache.org/v1alpha1 kind: IntegrationContext metadata: - name: root.integrationcontexts.camel.apache.org + name: core.integrationcontexts.camel.apache.org labels: app: "camel-k" camel.apache.org/context.created.by.kind: Operator camel.apache.org/context.created.by.name: core - camel.apache.org/context.type: platform \ No newline at end of file + camel.apache.org/context.type: platform +spec: + dependencies: + - camel:core \ No newline at end of file diff --git a/deploy/resources.go b/deploy/resources.go index e9525d6..ec79a89 100644 --- a/deploy/resources.go +++ b/deploy/resources.go @@ -2441,12 +2441,15 @@ status: apiVersion: camel.apache.org/v1alpha1 kind: IntegrationContext metadata: - name: root.integrationcontexts.camel.apache.org + name: core.integrationcontexts.camel.apache.org labels: app: "camel-k" camel.apache.org/context.created.by.kind: Operator camel.apache.org/context.created.by.name: core camel.apache.org/context.type: platform +spec: + dependencies: + - camel:core ` Resources["platform-integration-context-groovy.yaml"] = ` diff --git a/pkg/apis/camel/v1alpha1/types.go b/pkg/apis/camel/v1alpha1/types.go index 3a91be6..ee89870 100644 --- a/pkg/apis/camel/v1alpha1/types.go +++ b/pkg/apis/camel/v1alpha1/types.go @@ -125,6 +125,7 @@ type IntegrationContext struct { // IntegrationContextSpec -- type IntegrationContextSpec struct { Dependencies []string `json:"dependencies,omitempty"` + Classpath []string `json:"classpath,omitempty"` Configuration []ConfigurationSpec `json:"configuration,omitempty"` } diff --git a/pkg/apis/camel/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/camel/v1alpha1/zz_generated.deepcopy.go index 9feb415..dfd91cb 100644 --- a/pkg/apis/camel/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/camel/v1alpha1/zz_generated.deepcopy.go @@ -138,6 +138,11 @@ func (in *IntegrationContextSpec) DeepCopyInto(out *IntegrationContextSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.Classpath != nil { + in, out := &in.Classpath, &out.Classpath + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.Configuration != nil { in, out := &in.Configuration, &out.Configuration *out = make([]ConfigurationSpec, len(*in)) diff --git a/pkg/build/build_manager.go b/pkg/build/build_manager.go index 325e995..50d1187 100644 --- a/pkg/build/build_manager.go +++ b/pkg/build/build_manager.go @@ -57,7 +57,7 @@ func (m *Manager) Start(source Request) { resChannel := m.builder.Build(source) go func() { res := <-resChannel - m.builds.Store(res.Source.Identifier, &res) + m.builds.Store(res.Request.Identifier, &res) }() } @@ -69,7 +69,7 @@ func noBuildInfo() Result { func initialBuildInfo(source *Request) Result { return Result{ - Source: source, - Status: StatusStarted, + Request: source, + Status: StatusStarted, } } diff --git a/pkg/build/build_types.go b/pkg/build/build_types.go index a552aa1..0fd7309 100644 --- a/pkg/build/build_types.go +++ b/pkg/build/build_types.go @@ -30,7 +30,7 @@ type Identifier struct { Qualifier string } -// Source represent the integration code +// Request represent the integration code type Source struct { Name string Content string @@ -39,11 +39,11 @@ type Source struct { // Result represents the result of a build type Result struct { - Source *Request - Status Status - Image string - Error error - ResolvedDependencies []string + Request *Request + Status Status + Image string + Error error + Classpath []string } // Builder is supertype of all builders diff --git a/pkg/build/local/local_builder.go b/pkg/build/local/local_builder.go index 65e035f..2bb59ce 100644 --- a/pkg/build/local/local_builder.go +++ b/pkg/build/local/local_builder.go @@ -51,8 +51,8 @@ type localBuilder struct { } type buildOperation struct { - source build.Request - output chan build.Result + request build.Request + output chan build.Result } // NewLocalBuilder create a new builder @@ -65,11 +65,11 @@ func NewLocalBuilder(ctx context.Context, namespace string) build.Builder { return &builder } -func (b *localBuilder) Build(source build.Request) <-chan build.Result { +func (b *localBuilder) Build(request build.Request) <-chan build.Result { res := make(chan build.Result, 1) op := buildOperation{ - source: source, - output: res, + request: request, + output: res, } b.buffer <- op return res @@ -84,53 +84,57 @@ func (b *localBuilder) buildCycle(ctx context.Context) { case op := <-b.buffer: now := time.Now() logrus.Info("Starting new build") - image, err := b.execute(op.source) + res := b.execute(&op.request) elapsed := time.Now().Sub(now) - if err != nil { - logrus.Error("Error during build (total time ", elapsed.Seconds(), " seconds): ", err) - } else { - logrus.Info("Build completed in ", elapsed.Seconds(), " seconds") - } - if err != nil { - op.output <- build.Result{ - Source: &op.source, - Status: build.StatusError, - Error: err, - } + if res.Error != nil { + logrus.Error("Error during build (total time ", elapsed.Seconds(), " seconds): ", res.Error) } else { - op.output <- build.Result{ - Source: &op.source, - Status: build.StatusCompleted, - Image: image, - } + logrus.Info("Build completed in ", elapsed.Seconds(), " seconds") } + op.output <- res } } } -func (b *localBuilder) execute(source build.Request) (string, error) { - project, err := generateProject(source) +func (b *localBuilder) execute(request *build.Request) build.Result { + project, err := generateProject(request) if err != nil { - return "", err + return build.Result{ + Error: err, + Status: build.StatusError, + } } - tarFileName, err := maven.Build(project) + + res, err := maven.Build(project) if err != nil { - return "", err + return build.Result{ + Error: err, + Status: build.StatusError, + } } - logrus.Info("Created tar file ", tarFileName) + logrus.Info("Created tar file ", res.TarFilePath) - image, err := b.publish(tarFileName, source) + image, err := b.publish(res.TarFilePath, request) if err != nil { - return "", errors.Wrap(err, "could not publish docker image") + return build.Result{ + Error: errors.Wrap(err, "could not publish docker image"), + Status: build.StatusError, + } } - return image, nil + return build.Result{ + Request: request, + Image: image, + Error: nil, + Status: build.StatusCompleted, + Classpath: res.Classpath, + } } -func (b *localBuilder) publish(tarFile string, source build.Request) (string, error) { +func (b *localBuilder) publish(tarFile string, source *build.Request) (string, error) { bc := buildv1.BuildConfig{ TypeMeta: metav1.TypeMeta{ APIVersion: buildv1.SchemeGroupVersion.String(), @@ -253,7 +257,7 @@ func (b *localBuilder) publish(tarFile string, source build.Request) (string, er return is.Status.DockerImageRepository + ":" + source.Identifier.Qualifier, nil } -func generateProject(source build.Request) (maven.Project, error) { +func generateProject(source *build.Request) (maven.Project, error) { project := maven.Project{ XMLName: xml.Name{Local: "project"}, XMLNs: "http://maven.apache.org/POM/4.0.0", @@ -267,7 +271,7 @@ func generateProject(source build.Request) (maven.Project, error) { Dependencies: maven.Dependencies{ Dependencies: []maven.Dependency{ { - //TODO: camel version should be retrieved from an external source or provided as static version + //TODO: camel version should be retrieved from an external request or provided as static version GroupID: "org.apache.camel", ArtifactID: "camel-bom", Version: "2.22.1", diff --git a/pkg/build/local/local_builder_test.go b/pkg/build/local/local_builder_test.go index e6ec79b..ddb928e 100644 --- a/pkg/build/local/local_builder_test.go +++ b/pkg/build/local/local_builder_test.go @@ -25,7 +25,7 @@ import ( ) func TestProjectGeneration(t *testing.T) { - source := build.Request{ + request := build.Request{ Identifier: build.Identifier{ Name: "my-integration", Qualifier: "", @@ -43,7 +43,8 @@ func TestProjectGeneration(t *testing.T) { }, } - prj, err := generateProject(source) + prj, err := generateProject(&request) + assert.Nil(t, err) assert.NotNil(t, prj) assert.Equal(t, len(prj.Dependencies.Dependencies), 5) @@ -57,7 +58,7 @@ func TestProjectGeneration(t *testing.T) { } func TestProjectGenerationWithFailure(t *testing.T) { - source := build.Request{ + request := build.Request{ Identifier: build.Identifier{ Name: "my-integration", Qualifier: "", @@ -75,6 +76,7 @@ func TestProjectGenerationWithFailure(t *testing.T) { }, } - _, err := generateProject(source) + _, err := generateProject(&request) + assert.NotNil(t, err) } diff --git a/pkg/discover/dependencies_test.go b/pkg/discover/dependencies_test.go index ca5fff2..8e48cb1 100644 --- a/pkg/discover/dependencies_test.go +++ b/pkg/discover/dependencies_test.go @@ -18,14 +18,15 @@ limitations under the License. package discover import ( + "testing" + "github.com/apache/camel-k/pkg/apis/camel/v1alpha1" "github.com/stretchr/testify/assert" - "testing" ) func TestDependenciesJavaSource(t *testing.T) { code := v1alpha1.SourceSpec{ - Name: "Source.java", + Name: "Request.java", Language: v1alpha1.LanguageJavaSource, Content: ` from("telegram:bots/cippa").to("log:stash"); @@ -40,7 +41,7 @@ func TestDependenciesJavaSource(t *testing.T) { func TestDependenciesJavaClass(t *testing.T) { code := v1alpha1.SourceSpec{ - Name: "Source.class", + Name: "Request.class", Language: v1alpha1.LanguageJavaClass, Content: ` from("telegram:bots/cippa").to("log:stash"); @@ -54,7 +55,7 @@ func TestDependenciesJavaClass(t *testing.T) { func TestDependenciesJavaScript(t *testing.T) { code := v1alpha1.SourceSpec{ - Name: "source.js", + Name: "source.js", Language: v1alpha1.LanguageJavaScript, Content: ` from('telegram:bots/cippa').to("log:stash"); @@ -70,7 +71,7 @@ func TestDependenciesJavaScript(t *testing.T) { func TestDependenciesGroovy(t *testing.T) { code := v1alpha1.SourceSpec{ - Name: "source.groovy", + Name: "source.groovy", Language: v1alpha1.LanguageGroovy, Content: ` from('telegram:bots/cippa').to("log:stash"); @@ -82,4 +83,4 @@ func TestDependenciesGroovy(t *testing.T) { dependencies := Dependencies(code) // assert all dependencies are found and sorted (removing duplicates) assert.Equal(t, []string{"camel:amqp", "camel:core", "camel:telegram"}, dependencies) -} \ No newline at end of file +} diff --git a/pkg/discover/languages_test.go b/pkg/discover/languages_test.go index 919de9d..bbe0c45 100644 --- a/pkg/discover/languages_test.go +++ b/pkg/discover/languages_test.go @@ -18,14 +18,15 @@ limitations under the License. package discover import ( + "testing" + "github.com/apache/camel-k/pkg/apis/camel/v1alpha1" "github.com/stretchr/testify/assert" - "testing" ) func TestLanguageJavaSource(t *testing.T) { code := v1alpha1.SourceSpec{ - Name: "Source.java", + Name: "Request.java", } language := Language(code) assert.Equal(t, v1alpha1.LanguageJavaSource, language) @@ -33,7 +34,7 @@ func TestLanguageJavaSource(t *testing.T) { func TestLanguageAlreadySet(t *testing.T) { code := v1alpha1.SourceSpec{ - Name: "Source.java", + Name: "Request.java", Language: v1alpha1.LanguageJavaScript, } language := Language(code) diff --git a/pkg/stub/action/context/build.go b/pkg/stub/action/context/build.go index e8d26e4..6d06b22 100644 --- a/pkg/stub/action/context/build.go +++ b/pkg/stub/action/context/build.go @@ -73,6 +73,7 @@ func (action *integrationContextBuildAction) Handle(context *v1alpha1.Integratio target := context.DeepCopy() target.Status.Image = buildResult.Image target.Status.Phase = v1alpha1.IntegrationContextPhaseReady + target.Spec.Classpath = buildResult.Classpath if err := sdk.Update(target); err != nil { return err } diff --git a/pkg/stub/action/integration/deploy.go b/pkg/stub/action/integration/deploy.go index 8d873ea..c6ba592 100644 --- a/pkg/stub/action/integration/deploy.go +++ b/pkg/stub/action/integration/deploy.go @@ -19,8 +19,11 @@ package action import ( "fmt" + "os" "strings" + "github.com/apache/camel-k/version" + "github.com/apache/camel-k/pkg/apis/camel/v1alpha1" "github.com/operator-framework/operator-sdk/pkg/sdk" "github.com/pkg/errors" @@ -74,6 +77,12 @@ func getConfigMapFor(ctx *v1alpha1.IntegrationContext, integration *v1alpha1.Int // combine properties of integration with context, integration // properties have the priority properties := CombineConfigurationAsMap("property", ctx, integration) + classpath := make([]string, 0, len(ctx.Spec.Classpath)) + + //TODO: we need some constants + for _, path := range ctx.Spec.Classpath { + classpath = append(classpath, strings.Replace(path, "/tmp/artifacts/m2", "/deployments/m2", 1)) + } cm := corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ @@ -102,6 +111,7 @@ func getConfigMapFor(ctx *v1alpha1.IntegrationContext, integration *v1alpha1.Int Data: map[string]string{ "integration": integration.Spec.Source.Content, "properties": PropertiesString(properties), + "classpath": strings.Join(classpath, string(os.PathListSeparator)), }, } @@ -142,6 +152,11 @@ func getDeploymentFor(ctx *v1alpha1.IntegrationContext, integration *v1alpha1.In // set env vars needed by the runtime environment["JAVA_MAIN_CLASS"] = "org.apache.camel.k.jvm.Application" + environment["JAVA_LIB_DIR"] = "/etc/camel/conf" + + //TODO: remove this hack !!! s2i java (/opt/run-java/run-java.sh) fails if JAVA_APP_JAR is not set + environment["JAVA_APP_JAR"] = fmt.Sprintf("/deployments/m2/org/apache/camel/k/camel-k-runtime-jvm/%s/camel-k-runtime-jvm-%s.jar", version.Version, version.Version) + environment["CAMEL_K_ROUTES_URI"] = "file:/etc/camel/conf/" + sourceName environment["CAMEL_K_ROUTES_LANGUAGE"] = string(integration.Spec.Source.Language) environment["CAMEL_K_CONF"] = "/etc/camel/conf/application.properties" @@ -226,6 +241,9 @@ func getDeploymentFor(ctx *v1alpha1.IntegrationContext, integration *v1alpha1.In }, { Key: "properties", Path: "application.properties", + }, { + Key: "classpath", + Path: "classpath", }, }, }, diff --git a/pkg/util/maven/maven.go b/pkg/util/maven/maven.go index 1a919d2..8f67982 100644 --- a/pkg/util/maven/maven.go +++ b/pkg/util/maven/maven.go @@ -19,6 +19,7 @@ package maven import ( "archive/tar" + "bufio" "bytes" "encoding/xml" "io" @@ -38,38 +39,63 @@ const ( artifactDirPrefix = "maven-bin-" ) +// BuildResult -- +type BuildResult struct { + TarFilePath string + Classpath []string +} + // Build takes a project description and returns a binary tar with the built artifacts -func Build(project Project) (string, error) { +func Build(project Project) (BuildResult, error) { + res := BuildResult{} buildDir, err := ioutil.TempDir("", buildDirPrefix) if err != nil { - return "", errors.Wrap(err, "could not create temporary dir for maven source files") + return res, errors.Wrap(err, "could not create temporary dir for maven source files") } + defer os.RemoveAll(buildDir) err = createMavenStructure(buildDir, project) if err != nil { - return "", errors.Wrap(err, "could not write maven source files") + return res, errors.Wrap(err, "could not write maven source files") } - err = runMavenBuild(buildDir) + err = runMavenBuild(buildDir, &res) if err != nil { - return "", err + return res, err } - tarfile, err := createTar(buildDir, project) + + res.TarFilePath, err = createTar(project, &res) if err != nil { - return "", err + return res, err } - return tarfile, nil + + return res, nil } -func runMavenBuild(buildDir string) error { - copyDepsCmd := exec.Command("mvn", mavenExtraOptions(), "clean", "dependency:copy-dependencies") - copyDepsCmd.Dir = buildDir - copyDepsCmd.Stdout = os.Stdout - copyDepsCmd.Stderr = os.Stderr +func runMavenBuild(buildDir string, result *BuildResult) error { + // file where the classpath is listed + out := path.Join(buildDir, "integration.classpath") + + cmd := exec.Command("mvn", mavenExtraOptions(), "-Dmdep.outputFile="+out, "dependency:build-classpath") + cmd.Dir = buildDir + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + logrus.Infof("determine classpath: mvn: %v", cmd.Args) + if err := cmd.Run(); err != nil { + return errors.Wrap(err, "failure while determining classpath") + } - logrus.Infof("Copying maven dependencies: mvn %v", copyDepsCmd.Args) - if err := copyDepsCmd.Run(); err != nil { - return errors.Wrap(err, "failure while extracting maven dependencies") + lines, err := readLines(out) + if err != nil { + return err + } + + result.Classpath = make([]string, 0) + for _, line := range lines { + for _, item := range strings.Split(line, string(os.PathListSeparator)) { + result.Classpath = append(result.Classpath, item) + } } logrus.Info("Maven build completed successfully") @@ -83,7 +109,7 @@ func mavenExtraOptions() string { return "-Dcamel.noop=true" } -func createTar(buildDir string, project Project) (string, error) { +func createTar(project Project, result *BuildResult) (string, error) { artifactDir, err := ioutil.TempDir("", artifactDirPrefix) if err != nil { return "", errors.Wrap(err, "could not create temporary dir for maven artifacts") @@ -97,22 +123,15 @@ func createTar(buildDir string, project Project) (string, error) { defer tarFile.Close() writer := tar.NewWriter(tarFile) + defer writer.Close() - dependenciesDir := path.Join(buildDir, "target", "dependency") - dependencies, err := ioutil.ReadDir(dependenciesDir) - if err != nil { - return "", err - } - - for _, dep := range dependencies { - err = appendToTar(path.Join(dependenciesDir, dep.Name()), "", writer) + for _, path := range result.Classpath { + err = appendToTarWithPath(path, writer) if err != nil { return "", err } } - writer.Close() - return tarFileName, nil } @@ -146,6 +165,35 @@ func appendToTar(filePath string, tarPath string, writer *tar.Writer) error { return nil } +func appendToTarWithPath(path string, writer *tar.Writer) error { + info, err := os.Stat(path) + if err != nil { + return err + } + + //TODO: we ned some constants + relocatedPath := strings.TrimPrefix(path, "/tmp/artifacts") + + writer.WriteHeader(&tar.Header{ + Name: relocatedPath, + Size: info.Size(), + Mode: int64(info.Mode()), + ModTime: info.ModTime(), + }) + + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + _, err = io.Copy(writer, file) + if err != nil { + return errors.Wrap(err, "cannot add file to the tar archive") + } + return nil +} + func createMavenStructure(buildDir string, project Project) error { pom, err := GeneratePomFileContent(project) if err != nil { @@ -182,15 +230,20 @@ func writeFile(buildDir string, relativePath string, content string) error { return nil } -func envFileContent(env map[string]string) string { - if env == nil { - return "" +//TODO: move to a file utility package +func readLines(path string) ([]string, error) { + file, err := os.Open(path) + if err != nil { + return nil, err } - content := "" - for k, v := range env { - content = content + "export " + k + "=" + v + "\n" + defer file.Close() + + var lines []string + scanner := bufio.NewScanner(file) + for scanner.Scan() { + lines = append(lines, scanner.Text()) } - return content + return lines, scanner.Err() } // GeneratePomFileContent generate a pom.xml file from the given project definition diff --git a/pkg/util/maven/types.go b/pkg/util/maven/maven_project.go similarity index 100% rename from pkg/util/maven/types.go rename to pkg/util/maven/maven_project.go