Hadoop 写SequenceFile文件 源代码

  1. package com.tdxx.hadoop.sequencefile;
  2. import java.io.IOException;
  3. import org.apache.hadoop.conf.Configuration;
  4. import org.apache.hadoop.fs.FileSystem;
  5. import org.apache.hadoop.fs.Path;
  6. import org.apache.hadoop.io.IOUtils;
  7. import org.apache.hadoop.io.LongWritable;
  8. import org.apache.hadoop.io.SequenceFile;
  9. import org.apache.hadoop.io.SequenceFile.Writer;
  10. public class SequenceFileStudy{
  11.     /**
  12.      * 写入到sequence file
  13.      *
  14.      * @param filePath
  15.      * @param conf
  16.      * @param datas
  17.      */
  18.     public static void write2SequenceFile(String filePath, Configuration conf,
  19.             LongWritable key,LongWritable val) {
  20.         FileSystem fs = null;
  21.         Writer writer = null;
  22.         Path path = null;
  23.         //LongWritable idKey = new LongWritable(0);
  24.         try {
  25.             fs = FileSystem.get(conf);
  26.             path = new Path(filePath);
  27.             writer = SequenceFile.createWriter(fs, conf, path,
  28.                     LongWritable.class, LongWritable.class);
  29.             writer.append(key, val);
  30.         } catch (IOException e) {
  31.             // TODO Auto-generated catch block
  32.             e.printStackTrace();
  33.         } finally {
  34.             IOUtils.closeStream(writer);
  35.         }
  36.     }
  37.     private static Configuration getDefaultConf() {
  38.         Configuration conf = new Configuration();
  39.         conf.set(“mapred.job.tracker”, “local”);
  40.         conf.set(“fs.default.name”, “file:///”);
  41.         return conf;
  42.     }
  43.     /**
  44.      * @param args
  45.      */
  46.     public static void main(String[] args) {
  47.         String filePath = “data/longValue.sequence”; // 文件路径
  48.         // 生成数据
  49.         for (int i = 1; i <= 20; i++) {
  50.             // 写入到sequence file
  51.             LongWritable key = new LongWritable(i);
  52.             LongWritable value = new LongWritable(i+1);
  53.             write2SequenceFile(filePath, getDefaultConf(), key,value);
  54.         }
  55.     }
  56. }

标签