Apache Sqoop documentation¶

sqoop.sh tool repositoryencryption \
    -Forg.apache.sqoop.security.repo_encryption.password=old_password \
    -Forg.apache.sqoop.security.repo_encryption.hmac_algorithm=HmacSHA256 \
    -Forg.apache.sqoop.security.repo_encryption.cipher_algorithm=AES \
    -Forg.apache.sqoop.security.repo_encryption.cipher_key_size=16 \
    -Forg.apache.sqoop.security.repo_encryption.cipher_spec=AES/CBC/PKCS5Padding \
    -Forg.apache.sqoop.security.repo_encryption.initialization_vector_size=16 \
    -Forg.apache.sqoop.security.repo_encryption.pbkdf2_algorithm=PBKDF2WithHmacSHA1 \
    -Forg.apache.sqoop.security.repo_encryption.pbkdf2_rounds=4000 \
    -Torg.apache.sqoop.security.repo_encryption.password=new_password \
    -Torg.apache.sqoop.security.repo_encryption.hmac_algorithm=HmacSHA256 \
    -Torg.apache.sqoop.security.repo_encryption.cipher_algorithm=AES \
    -Torg.apache.sqoop.security.repo_encryption.cipher_key_size=16 \
    -Torg.apache.sqoop.security.repo_encryption.cipher_spec=AES/CBC/PKCS5Padding \
    -Torg.apache.sqoop.security.repo_encryption.initialization_vector_size=16 \
    -Torg.apache.sqoop.security.repo_encryption.pbkdf2_algorithm=PBKDF2WithHmacSHA1 \
    -Torg.apache.sqoop.security.repo_encryption.pbkdf2_rounds=4000

sqoop.sh tool repositoryencryption \
    -Forg.apache.sqoop.security.repo_encryption.password=password \
    -Forg.apache.sqoop.security.repo_encryption.hmac_algorithm=HmacSHA256 \
    -Forg.apache.sqoop.security.repo_encryption.cipher_algorithm=AES \
    -Forg.apache.sqoop.security.repo_encryption.cipher_key_size=16 \
    -Forg.apache.sqoop.security.repo_encryption.cipher_spec=AES/CBC/PKCS5Padding \
    -Forg.apache.sqoop.security.repo_encryption.initialization_vector_size=16 \
    -Forg.apache.sqoop.security.repo_encryption.pbkdf2_algorithm=PBKDF2WithHmacSHA1 \
    -Forg.apache.sqoop.security.repo_encryption.pbkdf2_rounds=4000 \
    -Torg.apache.sqoop.security.repo_encryption.password=password \
    -Torg.apache.sqoop.security.repo_encryption.hmac_algorithm=HmacSHA256 \
    -Torg.apache.sqoop.security.repo_encryption.cipher_algorithm=AES \
    -Torg.apache.sqoop.security.repo_encryption.cipher_key_size=16 \
    -Torg.apache.sqoop.security.repo_encryption.cipher_spec=AES/CBC/PKCS5Padding \
    -Torg.apache.sqoop.security.repo_encryption.initialization_vector_size=16 \
    -Torg.apache.sqoop.security.repo_encryption.pbkdf2_algorithm=PBKDF2WithHmacSHA1 \
    -Torg.apache.sqoop.security.repo_encryption.pbkdf2_rounds=4000

sqoop.sh tool repositoryencryption -FuseConf -TuseConf

sqoop.sh tool repositoryencryption \
    -Torg.apache.sqoop.security.repo_encryption.password=password \
    -Torg.apache.sqoop.security.repo_encryption.hmac_algorithm=HmacSHA256 \
    -Torg.apache.sqoop.security.repo_encryption.cipher_algorithm=AES \
    -Torg.apache.sqoop.security.repo_encryption.cipher_key_size=16 \
    -Torg.apache.sqoop.security.repo_encryption.cipher_spec=AES/CBC/PKCS5Padding \
    -Torg.apache.sqoop.security.repo_encryption.initialization_vector_size=16 \
    -Torg.apache.sqoop.security.repo_encryption.pbkdf2_algorithm=PBKDF2WithHmacSHA1 \
    -Torg.apache.sqoop.security.repo_encryption.pbkdf2_rounds=4000

sqoop.sh tool repositoryencryption -TuseConf

sqoop.sh tool repositoryencryption \
    -Forg.apache.sqoop.security.repo_encryption.password=password \
    -Forg.apache.sqoop.security.repo_encryption.hmac_algorithm=HmacSHA256 \
    -Forg.apache.sqoop.security.repo_encryption.cipher_algorithm=AES \
    -Forg.apache.sqoop.security.repo_encryption.cipher_key_size=16 \
    -Forg.apache.sqoop.security.repo_encryption.cipher_spec=AES/CBC/PKCS5Padding \
    -Forg.apache.sqoop.security.repo_encryption.initialization_vector_size=16 \
    -Forg.apache.sqoop.security.repo_encryption.pbkdf2_algorithm=PBKDF2WithHmacSHA1 \
    -Forg.apache.sqoop.security.repo_encryption.pbkdf2_rounds=4000

sqoop.sh tool repositoryencryption -FuseConf

sqoop2-server stop

sqoop2-tool upgrade

Tool class org.apache.sqoop.tools.tool.UpgradeTool has finished correctly.

sqoop2-server stop

org.apache.sqoop.repository.schema.immutable=false

org.apache.sqoop.connector.autoupgrade=true

org.apache.sqoop.driver.autoupgrade=true

sqoop2-server start

sqoop:000> help
For information about Sqoop, visit: http://sqoop.apache.org/

Available commands:
  exit    (\x  ) Exit the shell
  history (\H  ) Display, manage and recall edit-line history
  help    (\h  ) Display this help message
  set     (\st ) Configure various client options and settings
  show    (\sh ) Display various objects and configuration options
  create  (\cr ) Create new object in Sqoop repository
  delete  (\d  ) Delete existing object in Sqoop repository
  update  (\up ) Update objects in Sqoop repository
  clone   (\cl ) Create new object based on existing one
  start   (\sta) Start job
  stop    (\stp) Stop job
  status  (\stu) Display status of a job
  enable  (\en ) Enable object in Sqoop repository
  disable (\di ) Disable object in Sqoop repository

// create a placeholder for link
MLink link = client.createLink("connectorName");
link.setName("Vampire");
link.setCreationUser("Buffy");
MLinkConfig linkConfig = link.getConnectorLinkConfig();
// fill in the link config values
linkConfig.getStringInput("linkConfig.connectionString").setValue("jdbc:mysql://localhost/my");
linkConfig.getStringInput("linkConfig.jdbcDriver").setValue("com.mysql.jdbc.Driver");
linkConfig.getStringInput("linkConfig.username").setValue("root");
linkConfig.getStringInput("linkConfig.password").setValue("root");
// save the link object that was filled
Status status = client.saveLink(link);
if(status.canProceed()) {
 System.out.println("Created Link with Link Name : " + link.getName());
} else {
 System.out.println("Something went wrong creating the link");
}

String url = "http://localhost:12000/sqoop/";
SqoopClient client = new SqoopClient(url);
//Creating dummy job object
MJob job = client.createJob("fromLinkName", "toLinkName");
job.setName("Vampire");
job.setCreationUser("Buffy");
// set the "FROM" link job config values
MFromConfig fromJobConfig = job.getFromJobConfig();
fromJobConfig.getStringInput("fromJobConfig.schemaName").setValue("sqoop");
fromJobConfig.getStringInput("fromJobConfig.tableName").setValue("sqoop");
fromJobConfig.getStringInput("fromJobConfig.partitionColumn").setValue("id");
// set the "TO" link job config values
MToConfig toJobConfig = job.getToJobConfig();
toJobConfig.getStringInput("toJobConfig.outputDirectory").setValue("/usr/tmp");
// set the driver config values
MDriverConfig driverConfig = job.getDriverConfig();
driverConfig.getStringInput("throttlingConfig.numExtractors").setValue("3");

Status status = client.saveJob(job);
if(status.canProceed()) {
 System.out.println("Created Job with Job Name: "+ job.getName());
} else {
 System.out.println("Something went wrong creating the job");
}

printMessage(link.getConnectorLinkConfig().getConfigs());

private static void printMessage(List<MConfig> configs) {
  for(MConfig config : configs) {
    List<MInput<?>> inputlist = config.getInputs();
    if (config.getValidationMessages() != null) {
     // print every validation message
     for(Message message : config.getValidationMessages()) {
      System.out.println("Config validation message: " + message.getMessage());
     }
    }
    for (MInput minput : inputlist) {
      if (minput.getValidationStatus() == Status.WARNING) {
       for(Message message : minput.getValidationMessages()) {
        System.out.println("Config Input Validation Warning: " + message.getMessage());
      }
    }
    else if (minput.getValidationStatus() == Status.ERROR) {
      for(Message message : minput.getValidationMessages()) {
       System.out.println("Config Input Validation Error: " + message.getMessage());
      }
     }
    }
   }

private static final From FROM = new From(
      GenericJdbcFromInitializer.class,
      GenericJdbcPartitioner.class,
      GenericJdbcExtractor.class,
      GenericJdbcFromDestroyer.class);
...

@Override
public From getFrom() {
  return FROM;
}

private static final To TO = new To(
      GenericJdbcToInitializer.class,
      GenericJdbcLoader.class,
      GenericJdbcToDestroyer.class);
...

@Override
public To getTo() {
  return TO;
}

# Sqoop Foo Connector Properties
org.apache.sqoop.connector.class = org.apache.sqoop.connector.foo.FooConnector
org.apache.sqoop.connector.name = sqoop-foo-connector

 <dependencies>
  <!-- Sqoop modules -->
  <dependency>
    <groupId>org.apache.sqoop</groupId>
    <artifactId>connector-sdk</artifactId>
  </dependency>

  <!-- Testing specified modules -->
  <dependency>
    <groupId>org.testng</groupId>
    <artifactId>testng</artifactId>
    <scope>test</scope>
  </dependency>
  <dependency>
    <groupId>org.mockito</groupId>
    <artifactId>mockito-all</artifactId>
    <scope>test</scope>
  </dependency>
   <dependency>
     <groupId>org.apache.sqoop</groupId>
     <artifactId>sqoop-common-test</artifactId>
   </dependency>

   <dependency>
     <groupId>org.apache.sqoop</groupId>
     <artifactId>test</artifactId>
   </dependency>
  <!-- Connector required modules -->
  <dependency>
    <groupId>org.kitesdk</groupId>
    <artifactId>kite-data-core</artifactId>
  </dependency>
  ....
</dependencies>

<plugins>
  <plugin>
    <groupId>org.apache.maven.plugins</groupId>
    <artifactId>maven-assembly-plugin</artifactId>
    <version>${maven-assembly-plugin.version}</version>
    <dependencies>
      <dependency>
        <groupId>org.apache.sqoop</groupId>
        <artifactId>sqoop-assemblies</artifactId>
        <version>${sqoop.version}</version>
      </dependency>
    </dependencies>
    <executions>
      <execution>
        <id>make-assembly</id>
        <phase>package</phase>
        <goals>
          <goal>single</goal>
        </goals>
        <configuration>
          <finalName>${project.artifactId}-${project.version}</finalName>
          <appendAssemblyId>false</appendAssemblyId>
          <descriptorRefs>
            <descriptorRef>sqoop-connector</descriptorRef>
          </descriptorRefs>
        </configuration>
      </execution>
    </executions>
  </plugin>
</plugins>

# Generic JDBC Connector Properties
org.apache.sqoop.connector.class = org.apache.sqoop.connector.jdbc.GenericJdbcConnector
org.apache.sqoop.connector.name = generic-jdbc-connector

@Override
public Class getLinkConfigurationClass() {
  return LinkConfiguration.class;
}

@Override
public Class getJobConfigurationClass(Direction direction) {
  switch (direction) {
    case FROM:
      return FromJobConfiguration.class;
    case TO:
      return ToJobConfiguration.class;
    default:
      return null;
  }
}

@ConfigurationClass
public class LinkConfiguration {

  @Config public LinkConfig linkConfig;

  public LinkConfiguration() {
    linkConfig = new LinkConfig();
  }
}

@ConfigClass(validators = {@Validator(LinkConfig.ConfigValidator.class)})
public class LinkConfig {
  @Input(size = 128, validators = {@Validator(NotEmpty.class), @Validator(ClassAvailable.class)} )
  @Input(size = 128) public String jdbcDriver;
  @Input(size = 128) public String connectionString;
  @Input(size = 40)  public String username;
  @Input(size = 40, sensitive = true) public String password;
  @Input public Map<String, String> jdbcProperties;
}

# jdbc driver
connection.jdbcDriver.label = JDBC Driver Class
connection.jdbcDriver.help = Enter the fully qualified class name of the JDBC \
                   driver that will be used for establishing this connection.

# connect string
connection.connectionString.label = JDBC Connection String
connection.connectionString.help = Enter the value of JDBC connection string to be \
                   used by this connector for creating connections.

...

@Override
public ResourceBundle getBundle(Locale locale) {
  return ResourceBundle.getBundle(
  GenericJdbcConnectorConstants.RESOURCE_BUNDLE_NAME, locale);
}

@Input(size = 128, validators = {@Validator(value = StartsWith.class, strArg = "jdbc:")} )

@Input(size = 255, validators = { @Validator(NotEmpty.class) })

public class NotEmpty extends AbstractValidator<String> {
@Override
public void validate(String instance) {
  if (instance == null || instance.isEmpty()) {
   addMessage(Status.ERROR, "Can't be null nor empty");
  }
 }
}

 {
  id:7,
  inputs:[
    {
       id: 25,
       name: "throttlingConfig.numExtractors",
       type: "INTEGER",
       sensitive: false
    },
    {
       id: 26,
       name: "throttlingConfig.numLoaders",
       type: "INTEGER",
       sensitive: false
     }
  ],
  name: "throttlingConfig",
  type: "JOB"
}

link: {
      id : 3,
      name: "linkName",
      enabled: true,
      link-config-values: [{
          id: 4,
          inputs: [{
              id: 7,
              name: "linkConfig.jdbcDriver",
              value: "com.mysql.jdbc.Driver",
              type: "STRING",
              size: 128,
              sensitive: false
          }, {
              id: 8,
              name: "linkConfig.connectionString",
              value: "jdbc%3Amysql%3A%2F%2Fmysql.ent.cloudera.com%2Fsqoop",
              type: "STRING",
              size: 128,
              sensitive: false
          },
          ...
       }
     }

{
  "message":"DERBYREPO_0030:Unable to load specific job metadata from repository - Couldn't find job with id 2",
  "stack-trace":[
    {
      "file":"DerbyRepositoryHandler.java",
      "line":1111,
      "class":"org.apache.sqoop.repository.derby.DerbyRepositoryHandler",
      "method":"findJob"
    },
    {
      "file":"JdbcRepository.java",
      "line":451,
      "class":"org.apache.sqoop.repository.JdbcRepository$16",
      "method":"doIt"
    },
    {
      "file":"JdbcRepository.java",
      "line":90,
      "class":"org.apache.sqoop.repository.JdbcRepository",
      "method":"doWithConnection"
    },
    {
      "file":"JdbcRepository.java",
      "line":61,
      "class":"org.apache.sqoop.repository.JdbcRepository",
      "method":"doWithConnection"
    },
    {
      "file":"JdbcRepository.java",
      "line":448,
      "class":"org.apache.sqoop.repository.JdbcRepository",
      "method":"findJob"
    },
    {
      "file":"JobRequestHandler.java",
      "line":238,
      "class":"org.apache.sqoop.handler.JobRequestHandler",
      "method":"getJobs"
    }
  ],
  "class":"org.apache.sqoop.common.SqoopException"
}

{
   "id": 3,
   "validation-result": [
       {}
   ]
}

{
  "validation-result": [
    {
     "linkConfig": [
       {
         "message": "Invalid URI. URI must either be null or a valid URI. Here are a few valid example URIs: hdfs://example.com:8020/, hdfs://example.com/, file:///, file:///tmp, file://localhost/tmp",
         "status": "ERROR"
       }
     ]
   }
  ]
}

{
 source-url: "git://vbasavaraj.local/Users/vbasavaraj/Projects/SqoopRefactoring/sqoop2/common",
 source-revision: "418c5f637c3f09b94ea7fc3b0a4610831373a25f",
 build-version: "2.0.0-SNAPSHOT",
 api-versions: [
    "v1"
  ],
 user: "vbasavaraj",
 build-date: "Mon Nov 3 08:18:21 PST 2014"
}

{
  connectors: [{
      id: 1,
      link-config: [],
      job-config: {},
      name: "hdfs-connector",
      class: "org.apache.sqoop.connector.hdfs.HdfsConnector",
      all-config-resources: {},
      version: "2.0.0-SNAPSHOT"
  }, {
      id: 2,
      link-config: [],
      job-config: {},
      name: "generic-jdbc-connector",
      class: "org.apache.sqoop.connector.jdbc.GenericJdbcConnector",
      all-config - resources: {},
      version: "2.0.0-SNAPSHOT"
  }]
}

{
 connector: {
     id: 1,
     name: "connectorName",
     job-config: {
         TO: [{
             id: 3,
             inputs: [{
                 id: 3,
                 values: "TEXT_FILE,SEQUENCE_FILE",
                 name: "toJobConfig.outputFormat",
                 type: "ENUM",
                 sensitive: false
             }, {
                 id: 4,
                 values: "NONE,DEFAULT,DEFLATE,GZIP,BZIP2,LZO,LZ4,SNAPPY,CUSTOM",
                 name: "toJobConfig.compression",
                 type: "ENUM",
                 sensitive: false
             }, {
                 id: 5,
                 name: "toJobConfig.customCompression",
                 type: "STRING",
                 size: 255,
                 sensitive: false
             }, {
                 id: 6,
                 name: "toJobConfig.outputDirectory",
                 type: "STRING",
                 size: 255,
                 sensitive: false
             }],
             name: "toJobConfig",
             type: "JOB"
         }],
         FROM: [{
             id: 2,
             inputs: [{
                 id: 2,
                 name: "fromJobConfig.inputDirectory",
                 type: "STRING",
                 size: 255,
                 sensitive: false
             }],
             name: "fromJobConfig",
             type: "JOB"
         }]
     },
     link-config: [{
         id: 1,
         inputs: [{
             id: 1,
             name: "linkConfig.uri",
             type: "STRING",
             size: 255,
             sensitive: false
         }],
         name: "linkConfig",
         type: "LINK"
     }],
     name: "hdfs-connector",
     class: "org.apache.sqoop.connector.hdfs.HdfsConnector",
     all-config-resources: {
         fromJobConfig.label: "From Job configuration",
             toJobConfig.ignored.label: "Ignored",
             fromJobConfig.help: "Specifies information required to get data from Hadoop ecosystem",
             toJobConfig.ignored.help: "This value is ignored",
             toJobConfig.label: "ToJob configuration",
             toJobConfig.storageType.label: "Storage type",
             fromJobConfig.inputDirectory.label: "Input directory",
             toJobConfig.outputFormat.label: "Output format",
             toJobConfig.outputDirectory.label: "Output directory",
             toJobConfig.outputDirectory.help: "Output directory for final data",
             toJobConfig.compression.help: "Compression that should be used for the data",
             toJobConfig.outputFormat.help: "Format in which data should be serialized",
             toJobConfig.customCompression.label: "Custom compression format",
             toJobConfig.compression.label: "Compression format",
             linkConfig.label: "Link configuration",
             toJobConfig.customCompression.help: "Full class name of the custom compression",
             toJobConfig.storageType.help: "Target on Hadoop ecosystem where to store data",
             linkConfig.help: "Here you supply information necessary to connect to HDFS",
             linkConfig.uri.help: "HDFS URI used to connect to HDFS",
             linkConfig.uri.label: "HDFS URI",
             fromJobConfig.inputDirectory.help: "Directory that should be exported",
             toJobConfig.help: "You must supply the information requested in order to get information where you want to store your data."
     },
     version: "2.0.0-SNAPSHOT"
  }
}

{
   id: 3,
   job-config: [{
       id: 7,
       inputs: [{
           id: 25,
           name: "throttlingConfig.numExtractors",
           type: "INTEGER",
           sensitive: false
       }, {
           id: 26,
           name: "throttlingConfig.numLoaders",
           type: "INTEGER",
           sensitive: false
       }],
       name: "throttlingConfig",
       type: "JOB"
   }],
   all-config-resources: {
       throttlingConfig.numExtractors.label: "Extractors",
           throttlingConfig.numLoaders.help: "Number of loaders that Sqoop will use",
           throttlingConfig.numLoaders.label: "Loaders",
           throttlingConfig.label: "Throttling resources",
           throttlingConfig.numExtractors.help: "Number of extractors that Sqoop will use",
           throttlingConfig.help: "Set throttling boundaries to not overload your systems"
   },
   version: "1"
}

{
  links: [
    {
      id: 1,
      enabled: true,
      update-user: "root",
      link-config-values: [],
      name: "First Link",
      creation-date: 1415309361756,
      connector-name: "connectorName1",
      update-date: 1415309361756,
      creation-user: "root"
    },
    {
      id: 2,
      enabled: true,
      update-user: "root",
      link-config-values: [],
      name: "Second Link",
      creation-date: 1415309390807,
      connector-name: "connectorName2",
      update-date: 1415309390807,
      creation-user: "root"
    }
  ]
}

{
   link: {
       id: 1,
       enabled: true,
       link-config-values: [{
           id: 1,
           inputs: [{
               id: 1,
               name: "linkConfig.uri",
               value: "hdfs%3A%2F%2Fnamenode%3A8090",
               type: "STRING",
               size: 255,
               sensitive: false
           }],
           name: "linkConfig",
           type: "LINK"
       }],
       update-user: "root",
       name: "First Link",
       creation-date: 1415287846371,
       connector-name: "connectorName",
       update-date: 1415287846371,
       creation-user: "root"
   }
}

{
  link: {
      id: -1,
      enabled: true,
      link-config-values: [{
          id: 1,
          inputs: [{
              id: 1,
              name: "linkConfig.uri",
              value: "hdfs%3A%2F%2Fvbsqoop-1.ent.cloudera.com%3A8020%2Fuser%2Froot%2Fjob1",
              type: "STRING",
              size: 255,
              sensitive: false
          }],
          name: "testInput",
          type: "LINK"
      }],
      update-user: "root",
      name: "testLink",
      creation-date: 1415202223048,
      connector-name: "connectorName",
      update-date: 1415202223048,
      creation-user: "root"
  }
}

{
  "validation-result": [
      {
          "linkConfig": [
              {
                  "message": "Invalid URI. URI must either be null or a valid URI. Here are a few valid example URIs: hdfs://example.com:8020/, hdfs://example.com/, file:///, file:///tmp, file://localhost/tmp",
                  "status": "ERROR"
              }
          ]
      }
  ]
}

{
  "validation-result": [
      {}
  ]
}

{
   jobs: [{
      driver-config-values: [],
          enabled: true,
          from-connector-name: "fromConnectorName",
          update-user: "root",
          to-config-values: [],
          to-connector-name: "toConnectorName",
          creation-date: 1415310157618,
          update-date: 1415310157618,
          creation-user: "root",
          id: 1,
          to-link-name: "toLinkName",
          from-config-values: [],
          name: "First Job",
          from-link-name: "fromLinkName"
     },{
      driver-config-values: [],
          enabled: true,
          from-connector-name: "fromConnectorName",
          update-user: "root",
          to-config-values: [],
          to-connector-name: "toConnectorName",
          creation-date: 1415310650600,
          update-date: 1415310650600,
          creation-user: "root",
          id: 2,
          to-link-name: "toLinkName",
          from-config-values: [],
          name: "Second Job",
          from-link-name: "fromLinkName"
     }]
}

 {
   job: {
       driver-config-values: [{
               id: 7,
               inputs: [{
                   id: 25,
                   name: "throttlingConfig.numExtractors",
                   value: "3",
                   type: "INTEGER",
                   sensitive: false
               }, {
                   id: 26,
                   name: "throttlingConfig.numLoaders",
                   value: "3",
                   type: "INTEGER",
                   sensitive: false
               }],
               name: "throttlingConfig",
               type: "JOB"
           }],
           enabled: true,
           from-connector-name: "fromConnectorName",
           update-user: "root",
           to-config-values: [{
               id: 6,
               inputs: [{
                   id: 19,
                   name: "toJobConfig.schemaName",
                   type: "STRING",
                   size: 50,
                   sensitive: false
               }, {
                   id: 20,
                   name: "toJobConfig.tableName",
                   value: "text",
                   type: "STRING",
                   size: 2000,
                   sensitive: false
               }, {
                   id: 21,
                   name: "toJobConfig.sql",
                   type: "STRING",
                   size: 50,
                   sensitive: false
               }, {
                   id: 22,
                   name: "toJobConfig.columns",
                   type: "STRING",
                   size: 50,
                   sensitive: false
               }, {
                   id: 23,
                   name: "toJobConfig.stageTableName",
                   type: "STRING",
                   size: 2000,
                   sensitive: false
               }, {
                   id: 24,
                   name: "toJobConfig.shouldClearStageTable",
                   type: "BOOLEAN",
                   sensitive: false
               }],
               name: "toJobConfig",
               type: "JOB"
           }],
           to-connector-name: "toConnectorName",
           creation-date: 1415310157618,
           update-date: 1415310157618,
           creation-user: "root",
           id: 1,
           to-link-name: "toLinkName",
           from-config-values: [{
               id: 2,
               inputs: [{
                   id: 2,
                   name: "fromJobConfig.inputDirectory",
                   value: "hdfs%3A%2F%2Fvbsqoop-1.ent.cloudera.com%3A8020%2Fuser%2Froot%2Fjob1",
                   type: "STRING",
                   size: 255,
                   sensitive: false
               }],
               name: "fromJobConfig",
               type: "JOB"
           }],
           name: "First Job",
           from-link-name: "fromLinkName"
   }
}

{
  job: {
    driver-config-values: [
      {
        id: 7,
        inputs: [
          {
            id: 25,
            name: "throttlingConfig.numExtractors",
            value: "3",
            type: "INTEGER",
            sensitive: false
          },
          {
            id: 26,
            name: "throttlingConfig.numLoaders",
            value: "3",
            type: "INTEGER",
            sensitive: false
          }
        ],
        name: "throttlingConfig",
        type: "JOB"
      }
    ],
    enabled: true,
    from-connector-name: "fromConnectorName",
    update-user: "root",
    to-config-values: [
      {
        id: 6,
        inputs: [
          {
            id: 19,
            name: "toJobConfig.schemaName",
            type: "STRING",
            size: 50,
            sensitive: false
          },
          {
            id: 20,
            name: "toJobConfig.tableName",
            value: "text",
            type: "STRING",
            size: 2000,
            sensitive: false
          },
          {
            id: 21,
            name: "toJobConfig.sql",
            type: "STRING",
            size: 50,
            sensitive: false
          },
          {
            id: 22,
            name: "toJobConfig.columns",
            type: "STRING",
            size: 50,
            sensitive: false
          },
          {
            id: 23,
            name: "toJobConfig.stageTableName",
            type: "STRING",
            size: 2000,
            sensitive: false
          },
          {
            id: 24,
            name: "toJobConfig.shouldClearStageTable",
            type: "BOOLEAN",
            sensitive: false
          }
        ],
        name: "toJobConfig",
        type: "JOB"
      }
    ],
    to-connector-name: "toConnectorName",
    creation-date: 1415310157618,
    update-date: 1415310157618,
    creation-user: "root",
    id: -1,
    to-link-name: "toLinkName",
    from-config-values: [
      {
        id: 2,
        inputs: [
          {
            id: 2,
            name: "fromJobConfig.inputDirectory",
            value: "hdfs%3A%2F%2Fvbsqoop-1.ent.cloudera.com%3A8020%2Fuser%2Froot%2Fjob1",
            type: "STRING",
            size: 255,
            sensitive: false
          }
        ],
        name: "fromJobConfig",
        type: "JOB"
      }
    ],
    name: "Test Job",
    from-link-name: "fromLinkName"
   }
 }

{
  "validation-result": [
      {
          "linkConfig": [
              {
                  "message": "Invalid URI. URI must either be null or a valid URI. Here are a few valid example URIs: hdfs://example.com:8020/, hdfs://example.com/, file:///, file:///tmp, file://localhost/tmp",
                  "status": "ERROR"
              }
          ]
      }
  ]
}

{
  "validation-result": [
      {}
  ]
}

{
  "submission": {
    "progress": -1,
    "last-update-date": 1415312531188,
    "external-id": "job_1412137947693_0004",
    "status": "BOOTING",
    "job": 2,
    "job-name": "jobName",
    "creation-date": 1415312531188,
    "to-schema": {
      "created": 1415312531426,
      "name": "HDFS file",
      "columns": []
    },
    "external-link": "http://vbsqoop-1.ent.cloudera.com:8088/proxy/application_1412137947693_0004/",
    "from-schema": {
      "created": 1415312531342,
      "name": "text",
      "columns": [
        {
          "name": "id",
          "nullable": true,
          "unsigned": null,
          "type": "FIXED_POINT",
          "size": null
        },
        {
          "name": "txt",
          "nullable": true,
          "type": "TEXT",
          "size": null
        }
      ]
    }
  }
}

{
  submission: {
    progress: -1,
    last-update-date: 1415312809485,
    external-id: "job_1412137947693_0004",
    status: "SUCCEEDED",
    job: 2,
    job-name: "jobName",
    creation-date: 1415312531188,
    external-link: "http://vbsqoop-1.ent.cloudera.com:8088/proxy/application_1412137947693_0004/",
    counters: {
      org.apache.hadoop.mapreduce.JobCounter: {
        SLOTS_MILLIS_MAPS: 373553,
        MB_MILLIS_MAPS: 382518272,
        TOTAL_LAUNCHED_MAPS: 10,
        MILLIS_MAPS: 373553,
        VCORES_MILLIS_MAPS: 373553,
        OTHER_LOCAL_MAPS: 10
      },
      org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter: {
        BYTES_WRITTEN: 0
      },
      org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter: {
        BYTES_READ: 0
      },
      org.apache.hadoop.mapreduce.TaskCounter: {
        MAP_INPUT_RECORDS: 0,
        MERGED_MAP_OUTPUTS: 0,
        PHYSICAL_MEMORY_BYTES: 4065599488,
        SPILLED_RECORDS: 0,
        COMMITTED_HEAP_BYTES: 3439853568,
        CPU_MILLISECONDS: 236900,
        FAILED_SHUFFLE: 0,
        VIRTUAL_MEMORY_BYTES: 15231422464,
        SPLIT_RAW_BYTES: 1187,
        MAP_OUTPUT_RECORDS: 1000000,
        GC_TIME_MILLIS: 7282
      },
      org.apache.hadoop.mapreduce.FileSystemCounter: {
        FILE_WRITE_OPS: 0,
        FILE_READ_OPS: 0,
        FILE_LARGE_READ_OPS: 0,
        FILE_BYTES_READ: 0,
        HDFS_BYTES_READ: 1187,
        FILE_BYTES_WRITTEN: 1191230,
        HDFS_LARGE_READ_OPS: 0,
        HDFS_WRITE_OPS: 10,
        HDFS_READ_OPS: 10,
        HDFS_BYTES_WRITTEN: 276389736
      },
      org.apache.sqoop.submission.counter.SqoopCounters: {
        ROWS_READ: 1000000
      }
    }
  }
}

{
  "submission": {
    "progress": -1,
    "last-update-date": 1415312390570,
    "status": "FAILURE_ON_SUBMIT",
    "error-summary": "org.apache.sqoop.common.SqoopException: GENERIC_HDFS_CONNECTOR_0000:Error occurs during partitioner run",
    "job": 1,
    "job-name": "jobName",
    "creation-date": 1415312390570,
    "to-schema": {
      "created": 1415312390797,
      "name": "text",
      "columns": [
        {
          "name": "id",
          "nullable": true,
          "unsigned": null,
          "type": "FIXED_POINT",
          "size": null
        },
        {
          "name": "txt",
          "nullable": true,
          "type": "TEXT",
          "size": null
        }
      ]
    },
    "from-schema": {
      "created": 1415312390778,
      "name": "HDFS file",
      "columns": [
      ]
    },
    "error-details": "org.apache.sqoop.common.SqoopException: GENERIC_HDFS_CONNECTOR_00"
  }
}

{
    "submission": {
        "progress": 0.25,
        "last-update-date": 1415312603838,
        "external-id": "job_1412137947693_0004",
        "status": "RUNNING",
        "job": 2,
        "job-name": "jobName",
        "creation-date": 1415312531188,
        "external-link": "http://vbsqoop-1.ent.cloudera.com:8088/proxy/application_1412137947693_0004/"
    }
}

{
  submissions: [
    {
      progress: -1,
      last-update-date: 1415312809485,
      external-id: "job_1412137947693_0004",
      status: "SUCCEEDED",
      job: 2,
      job-name: "jobName",
      creation-date: 1415312531188,
      external-link: "http://vbsqoop-1.ent.cloudera.com:8088/proxy/application_1412137947693_0004/",
      counters: {
        org.apache.hadoop.mapreduce.JobCounter: {
          SLOTS_MILLIS_MAPS: 373553,
          MB_MILLIS_MAPS: 382518272,
          TOTAL_LAUNCHED_MAPS: 10,
          MILLIS_MAPS: 373553,
          VCORES_MILLIS_MAPS: 373553,
          OTHER_LOCAL_MAPS: 10
        },
        org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter: {
          BYTES_WRITTEN: 0
        },
        org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter: {
          BYTES_READ: 0
        },
        org.apache.hadoop.mapreduce.TaskCounter: {
          MAP_INPUT_RECORDS: 0,
          MERGED_MAP_OUTPUTS: 0,
          PHYSICAL_MEMORY_BYTES: 4065599488,
          SPILLED_RECORDS: 0,
          COMMITTED_HEAP_BYTES: 3439853568,
          CPU_MILLISECONDS: 236900,
          FAILED_SHUFFLE: 0,
          VIRTUAL_MEMORY_BYTES: 15231422464,
          SPLIT_RAW_BYTES: 1187,
          MAP_OUTPUT_RECORDS: 1000000,
          GC_TIME_MILLIS: 7282
        },
        org.apache.hadoop.mapreduce.FileSystemCounter: {
          FILE_WRITE_OPS: 0,
          FILE_READ_OPS: 0,
          FILE_LARGE_READ_OPS: 0,
          FILE_BYTES_READ: 0,
          HDFS_BYTES_READ: 1187,
          FILE_BYTES_WRITTEN: 1191230,
          HDFS_LARGE_READ_OPS: 0,
          HDFS_WRITE_OPS: 10,
          HDFS_READ_OPS: 10,
          HDFS_BYTES_WRITTEN: 276389736
        },
        org.apache.sqoop.submission.counter.SqoopCounters: {
          ROWS_READ: 1000000
        }
      }
    },
    {
      progress: -1,
      last-update-date: 1415312390570,
      status: "FAILURE_ON_SUBMIT",
      error-summary: "org.apache.sqoop.common.SqoopException: GENERIC_HDFS_CONNECTOR_0000:Error occurs during partitioner run",
      job: 1,
      job-name: "jobName",
      creation-date: 1415312390570,
      error-details: "org.apache.sqoop.common.SqoopException: GENERIC_HDFS_CONNECTOR_0000:Error occurs during partitioner...."
    }
  ]
}

{
  role: {
      name: "testRole",
  }
}

{
  roles: [{
      name: "testRole",
  }],
  principals: [{
      name: "testPrincipalName",
      type: "USER",
  }]
}

{
  privileges: [{
      resource-name: "testResourceName",
      resource-type: "LINK",
      action: "READ",
      with-grant-option: false,
  }]
  principals: [{
      name: "testPrincipalName",
      type: "USER",
  }]
}

org.apache.sqoop.authentication.type=SIMPLE
org.apache.sqoop.authentication.handler=org.apache.sqoop.security.authentication.SimpleAuthenticationHandler
org.apache.sqoop.anonymous=true

<Sqoop Folder>/bin/sqoop.sh server start

<Sqoop Folder>/bin/sqoop.sh client

addprinc -randkey HTTP/<FQDN>@<REALM>
addprinc -randkey sqoop/<FQDN>@<REALM>
xst -k /home/kerberos/sqoop.keytab HTTP/<FQDN>@<REALM>
xst -k /home/kerberos/sqoop.keytab sqoop/<FQDN>@<REALM>

org.apache.sqoop.authentication.type=KERBEROS
org.apache.sqoop.authentication.handler=org.apache.sqoop.security.authentication.KerberosAuthenticationHandler
org.apache.sqoop.authentication.kerberos.principal=sqoop/_HOST@<REALM>
org.apache.sqoop.authentication.kerberos.keytab=/home/kerberos/sqoop.keytab
org.apache.sqoop.authentication.kerberos.http.principal=HTTP/_HOST@<REALM>
org.apache.sqoop.authentication.kerberos.http.keytab=/home/kerberos/sqoop.keytab
org.apache.sqoop.authentication.kerberos.proxyuser=true

export SQOOP2_HOST=$(hostname -f).

sudo –u sqoop <Sqoop Folder>/bin/sqoop.sh server start

kinit HTTP/<FQDN>@<REALM> -kt /home/kerberos/sqoop.keytab

<Sqoop Folder>/bin/sqoop.sh client

2014-12-04 15:02:58,038 INFO  security.KerberosAuthenticationHandler [org.apache.sqoop.security.authentication.KerberosAuthenticationHandler.secureLogin(KerberosAuthenticationHandler.java:84)] Using Kerberos authentication, principal [sqoop/_HOST@HADOOP.COM] keytab [/home/kerberos/sqoop.keytab]

Refreshing Kerberos configuration
Acquire TGT from Cache
Principal is HTTP/<FQDN>@HADOOP.COM
null credentials from Ticket Cache
principal is HTTP/<FQDN>@HADOOP.COM
Will use keytab
Commit Succeeded

<User>: <Groups>
user_all: group1, group2
user1: group1
user2: group2

CREATE ROLE –role role_name
DROP ROLE –role role_name
SHOW ROLE

GRANT ROLE --principal-type principal_type --principal principal_name --role role_name
REVOKE ROLE --principal-type principal_type --principal principal_name --role role_name
SHOW ROLE --principal-type principal_type --principal principal_name
SHOW PRINCIPAL --role role_name

GRANT PRIVILEGE --principal-type principal_type --principal principal_name --resource-type resource_type --resource resource_name --action action_name [--with-grant]
REVOKE PRIVILEGE --principal-type principal_type --principal principal_name [--resource-type resource_type --resource resource_name --action action_name] [--with-grant]
SHOW PRIVILEGE –principal-type principal_type –principal principal_name [--resource-type resource_type --resource resource_name --action action_name]