BigData Important Code Lines

Question 1

// Filter function in Scala
val lines = sc.textFile(…)
lines.filter(x => x.contains(“ERROR”)).count()

// Filter function in Java
JavaRDD<String> lines = sc.textFile(...);
lines.filter(new Function<String, Boolean>() {
Boolean call(String s) { return s.contains("ERROR"); }}).count();</String>

// WordCount in R
words <- flatMap(textFile, function(line){strsplit(line, “ “)[[1]]})
wordCount <- lapply(words, function(word){list(word, 1L)})
counts <- reduceByKey(wordCount, “+”, 2L)

// WordCount in Python
text_file = sc.textFile(“hdfs://…”)
counts = text_file.flatMap(lambda line: line.split(“ “)) \
.map(lambda word: (word, 1)) \
.reduceByKey(lambda a, b: a + b)