Demo entry 6717475

java

   

Submitted by anonymous on Feb 25, 2018 at 04:06
Language: Java. Code size: 1.6 kB.

package com.zs;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

/**
 * @author hangchaoli
 * @Description 配置hadoop集群环境,配置MR执行环境信息,配置文件输入输出路径
 */
public class MRConf {
    public static void main(String[] args) throws IOException {
        Configuration conf = new Configuration();
        // IDE环境Run需配置jar包地址
        conf.set("mapred.jar", "d:\\mapred.jar");
        // jobtracker配置地址,MR执行
        conf.set("mapred.job.tracker", "master:9001");
        // 创建任务
        Job job = new Job(conf, "MRConf");
        job.setJarByClass(MRConf.class);
        job.setMapperClass(goMapper.class);
        job.setReducerClass(goReducer.class);
        // 设置输入路径,文件解析
        FileInputFormat.addInputPath(job, new Path("hdfs://master:9000/20180203/zs.txt"));
        // 设置输出路径,保存解析结果
        FileOutputFormat.setOutputPath(job, new Path("hdfs://master:9000/20180203/result"));
        // MR输入输出参数类型,map函数的输出对应到reduce函数的输入
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);
        try {
            // 任务提交
            System.exit(job.waitForCompletion(true) ? 0 : 1);
        } catch (ClassNotFoundException e) {
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
}

This snippet took 0.01 seconds to highlight.

Back to the Entry List or Home.

Delete this entry (admin only).