scala – 创建像TextLine一样的Scalding Source,它将多个文件组

创建自己的输入格式,扩展CombineFileInputFormat并使用您自己的自定义RecordReader.我正在向您展示Java代码,但如果您愿意,可以轻松地将其转换为Scala.

import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.LineRecordReader;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.lib.CombineFileInputFormat;
import org.apache.hadoop.mapred.lib.CombineFileRecordReader;
import org.apache.hadoop.mapred.lib.CombineFileSplit;

public class CombinedInputFormat<K,V> extends CombineFileInputFormat<K,V> {

    public static class MyKeyValueLineRecordReader implements RecordReader<LongWritable,Text> {
        private final RecordReader<LongWritable,Text> delegate;

        public MyKeyValueLineRecordReader(CombineFileSplit split,Configuration conf,Reporter reporter,Integer idx) throws IOException {
            FileSplit fileSplit = new FileSplit(split.getPath(idx),split.getOffset(idx),split.getLength(idx),split.getLocations());
            delegate = new LineRecordReader(conf,fileSplit);
        }

        @Override
        public boolean next(LongWritable key,Text value) throws IOException {
            return delegate.next(key,value);
        }

        @Override
        public LongWritable createKey() {
            return delegate.createKey();
        }

        @Override
        public Text createValue() {
            return delegate.createValue();
        }

        @Override
        public long getPos() throws IOException {
            return delegate.getPos();
        }

        @Override
        public void close() throws IOException {
            delegate.close();
        }

        @Override
        public float getProgress() throws IOException {
            return delegate.getProgress();
        }
    }

    @Override
    public RecordReader getRecordReader(InputSplit split,JobConf job,Reporter reporter) throws IOException {
        return new CombineFileRecordReader(job,(CombineFileSplit) split,reporter,(Class) MyKeyValueLineRecordReader.class);
    }

}

然后,您需要扩展TextLine类并使其使用您刚刚定义的自己的输入格式(从现在开始的Scala代码).

import cascading.scheme.hadoop.TextLine
import cascading.flow.FlowProcess
import org.apache.hadoop.mapred.{OutputCollector,RecordReader,JobConf}
import cascading.tap.Tap
import com.twitter.scalding.{FixedPathSource,TextLineScheme}
import cascading.scheme.Scheme

class CombineFileTextLine extends TextLine{

  override def sourceConfInit(flowProcess: FlowProcess[JobConf],tap: Tap[JobConf,RecordReader[_,_],OutputCollector[_,_]],conf: JobConf) {
    super.sourceConfInit(flowProcess,tap,conf)
    conf.setInputFormat(classOf[CombinedInputFormat[String,String]])
  }
}

为您的组合输入创建一个方案.

trait CombineFileTextLineScheme extends TextLineScheme{

  override def hdfsScheme = new CombineFileTextLine().asInstanceOf[Scheme[JobConf,_,_]]
}

最后,创建您的源类：

case class CombineFileMultipleTextLine(p : String*) extends  FixedPathSource(p :_*) with CombineFileTextLineScheme

如果要使用单个路径而不是多个路径,则对源类的更改是微不足道的.

我希望有所帮助.