Need an example POM file for running a Pentaho kitchen/pan transformation and job during the integration test phase

I have spent too many hours on Google and on the Pentaho forums looking for a simple example for what I would think would be common use case. I am asking fellow SE folks to help answer this question.

I have a project which is a mixture of Java and Pentaho ETL jobs. We would like to build, test, and deploy both types of projects using maven 3.

I am looking for an example POM that will execute the transformations and jobs (using kitchen/pan, I assume) during the integration test phase. In addition, if possible, an example of what you are doing to test the kettle jobs within the database (e.g. DBunit). So far here is what I have for information.

The user docs for kitchen which shows how to run a job on the command line in batch file: http://wiki.pentaho.com/display/EAI/Kitchen+User+Documentation
The user docs for PAN which shows how to run a transformation on the command line in batch file: http://wiki.pentaho.com/display/EAI/Pan+User+Documentation
While I could use exec plugin(http://mojo.codehaus.org/exec-maven-plugin/) to execute a batch file which invokes kitchen and pan, that seems like a hack at best. I am looking for a more "native" maven approach

A work in progress POM file:

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <groupId>pentaho-example</groupId>
    <artifactId>pentaho-example</artifactId>
    <version>1.0-SNAPSHOT</version>
    <packaging>jar</packaging>
    <name>pentaho-example</name>
    <!--
        required by pentaho stuff to bring in the needed jars to run via
        command line for testing
    -->
    <repositories>
        <repository>
            <releases>
                <updatePolicy>always</updatePolicy>
                <checksumPolicy>warn</checksumPolicy>
            </releases>
            <snapshots>
                <enabled>false</enabled>
                <checksumPolicy>warn</checksumPolicy>
            </snapshots>
            <id>pentaho-repo</id>
            <url>http://repo.pentaho.org/artifactory/pentaho/</url>
        </repository>
        <repository>
            <id>pentaho-third-party</id>
            <url>http://repo.pentaho.org/artifactory/third-party/</url>
        </repository>
    </repositories>
    <dependencies>
        <dependency>
            <groupId>com.mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.7</version>
        </dependency>
        <dependency>
            <groupId>org.pentaho</groupId>
            <artifactId>core</artifactId>
            <version>4.2.1</version>
        </dependency>
    </dependencies>
    <build>
        <plugins>
            <plugin>
                <artifactId>maven-assembly-plugin</artifactId>
                <version>2.2.2</version>
                <configuration>
                    <descriptorRefs>
                        <descriptorRef>project</descriptorRef>
                    </descriptorRefs>
                    <!--
                        <descriptors> <descriptor>src/assemble/etl-only.xml</descriptor>
                        </descriptors>
                    -->
                </configuration>
                <executions>
                    <execution>
                        <id>make-assembly</id> <!-- this is used for inheritance merges -->
                        <phase>package</phase> <!-- bind to the packaging phase -->
                        <goals>
                            <goal>single</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>
</project>

Does anyone have more information or an example POM to show how this can be done?

TIA,

Scott

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <groupId>module-groupId</groupId>
    <artifactId>module-artifact-id</artifactId>
    <version>2.4.6-SNAPSHOT</version>
    <packaging>pom</packaging>
    <name>module-name</name>

    <modules>
        <module>...</module>
        <module>...</module>
        <module>...</module>
    </modules>

    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <maven.compiler.source>1.8</maven.compiler.source>
        <maven.compiler.target>1.8</maven.compiler.target>
        <eula-wrap_create-dist-phase />
        <eula-wrap_assign-deps-to-properties-phase />
        <mockito.version>1.10.19</mockito.version>
        <pentaho-metadata.version>7.1.0.0-12</pentaho-metadata.version>
        <eula-wrap_create-izpack-installer-jar-phase />
        <pdi.version>7.1.0.0-12</pdi.version>
        <eula-wrap_attach-dist-phase />
        <junit.version>4.12</junit.version>
        <jersey.version>1.19.1</jersey.version>
        <jsr311-api.version>1.1.1</jsr311-api.version>
    </properties>

    <dependencies>
        <!-- https://public.nexus.pentaho.org/content/groups/omni/ -->
        <dependency>
            <groupId>pentaho</groupId>
            <artifactId>mondrian</artifactId>
            <version>3.12.0.1-196</version>
        </dependency>
        <!-- https://public.nexus.pentaho.org/content/groups/omni/ -->
        <dependency>
            <groupId>pentaho-kettle</groupId>
            <artifactId>kettle-core</artifactId>
            <version>${pdi.version}</version>
            <exclusions>
                <exclusion>
                    <artifactId>xercesImpl</artifactId>
                    <groupId>xerces</groupId>
                </exclusion>
                <exclusion>
                    <artifactId>commons-httpclient</artifactId>
                    <groupId>commons-httpclient</groupId>
                </exclusion>
            </exclusions>
        </dependency>
        <dependency>
            <groupId>pentaho-kettle</groupId>
            <artifactId>kettle-engine</artifactId>
            <version>${pdi.version}</version>
        </dependency>
        <!--<dependency>
            <groupId>org.pentaho.di.plugins</groupId>
        <artifactId>pdi-core-plugins-impl</artifactId>
            <version>${pdi.version}</version>
        </dependency> -->
        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpclient</artifactId>
            <version>4.5.3</version>
        </dependency>
        <dependency>
            <groupId>org.mockito</groupId>
            <artifactId>mockito-all</artifactId>
            <version>${mockito.version}</version>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>xerces</groupId>
            <artifactId>xercesImpl</artifactId>
            <version>2.9.1</version>
            <scope>test</scope>
            <exclusions>
                <exclusion>
                    <artifactId>*</artifactId>
                    <groupId>*</groupId>
                </exclusion>
            </exclusions>
        </dependency>
        <!-- https://mvnrepository.com/artifact/xml-apis/xml-apis -->
        <dependency>
            <groupId>xml-apis</groupId>
            <artifactId>xml-apis</artifactId>
            <version>1.4.01</version>
        </dependency>


        <!-- https://mvnrepository.com/artifact/com.google.code.gson/gson -->
        <dependency>
            <groupId>com.google.code.gson</groupId>
            <artifactId>gson</artifactId>
            <version>2.8.0</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/com.google.guava/guava -->
        <dependency>
            <groupId>com.google.guava</groupId>
            <artifactId>guava</artifactId>
            <version>25.0-jre</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml -->
        <dependency>
            <groupId>org.apache.poi</groupId>
            <artifactId>poi-ooxml</artifactId>
            <version>4.0.0</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.quartz-scheduler/quartz -->
        <dependency>
            <groupId>org.quartz-scheduler</groupId>
            <artifactId>quartz</artifactId>
            <version>2.2.1</version>
        </dependency>


        <!-- https://mvnrepository.com/artifact/org.apache.poi/poi -->
        <dependency>
            <groupId>org.apache.poi</groupId>
            <artifactId>poi</artifactId>
            <version>4.0.0</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-lang3</artifactId>
            <version>3.4</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-collections4 -->
        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-collections4</artifactId>
            <version>4.1</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-math3 -->
        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-math3</artifactId>
            <version>3.6.1</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-io -->
        <dependency>
            <groupId>commons-io</groupId>
            <artifactId>commons-io</artifactId>
            <version>1.3.2</version>
        </dependency>


        <!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-core -->
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-core</artifactId>
            <version>2.11.0</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-api -->
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-api</artifactId>
            <version>2.11.0</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-slf4j-impl -->
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-slf4j-impl</artifactId>
            <version>2.11.0</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-compress -->
        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-compress</artifactId>
            <version>1.18</version>
        </dependency>

    </dependencies>


    <!-- Build Settings -->
    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-release-plugin</artifactId>
                <version>2.4.2</version>
                <configuration>
                    <tagNameFormat>v@{project.version}</tagNameFormat>
                    <autoVersionSubmodules>true</autoVersionSubmodules>
                    <releaseProfiles>releases</releaseProfiles>
                </configuration>
            </plugin>
        </plugins>
    </build>
    <profiles>
        <profile>
            <id>releases</id>
            <build>
                <plugins>
                    <plugin>
                        <groupId>org.apache.maven.plugins</groupId>
                        <artifactId>maven-deploy-plugin</artifactId>
                        <version>2.8.2</version>
                        <configuration>
                            <skip>true</skip>
                        </configuration>
                    </plugin>
                </plugins>
            </build>
        </profile>
    </profiles>
</project>

Java example

import java.util.logging.Level;
import java.util.logging.Logger;

import org.pentaho.di.core.KettleEnvironment;
import org.pentaho.di.job.Job;
import org.pentaho.di.job.JobMeta;
import org.pentaho.di.repository.Repository;

public class KettleJobExec {

    private static final Logger logger = Logger.getLogger(KettleJobExec.class.getName());

    public static void runJob(String jobPath) {
        try {
            //System.setProperty("KETTLE_HOME",PropertyManager.getInstance().getPropertyAsString(Constantes.KETTLE_HOME));
            Repository repository = null;

            logger.log(Level.INFO, "Kettle enviroment init");
            KettleEnvironment.init(false);

            logger.log(Level.INFO, "JobMeta creation: " + jobPath);
            JobMeta jobMeta = new JobMeta(jobPath, repository);

            logger.log(Level.INFO, "Job creation");
            Job job = new Job(repository, jobMeta);

            logger.log(Level.INFO, "Job start");
            job.start();

            logger.log(Level.INFO, "Job wait until finished");
            job.waitUntilFinished();

            if (job.getErrors() > 0) {
                throw new Exception("Error Executing Job: " + job.getStatus());
            }

            
        } catch (Exception e) {
            logger.log(Level.SEVERE, e.getMessage(), e);
        } finally {
            logger.log(Level.INFO, "Kettle enviroment shutdown");
            KettleEnvironment.shutdown();
        }
    }

}

Recommended topics

Hot tags