pythonspark 写入csv_将Spark DataFrame的内容保存为一个CSV文件

对于那些仍想做到这一点这里就是我得到了它使用的火花2.1斯卡拉一些java.nio.file帮助下完成的。

val df: org.apache.spark.sql.DataFrame = ??? // data frame to write

val file: java.nio.file.Path = ??? // target output file (i.e. 'out.csv')

import scala.collection.JavaConversions._

// write csv into temp directory which contains the additional spark output files

// could use Files.createTempDirectory instead

val tempDir = file.getParent.resolve(file.getFileName + "_tmp")

df.coalesce(1)

.write.format("com.databricks.spark.csv")

.option("header", "true")

.save(tempDir.toAbsolutePath.toString)

// find the actual csv file

val tmpCsvFile = Files.walk(tempDir, 1).iterator().toSeq.find { p =>

val fname = p.getFileName.toString

fname.startsWith("part-00000") && fname.endsWith(".csv") && Files.isRegularFile(p)

}.get

// move to desired final path

Files.move(tmpCsvFile, file)

// delete temp directory

Files.walk(tempDir)

.sorted(java.util.Comparator.reverseOrder())

.iterator().toSeq

.foreach(Files.delete(_))