1

我正在读取一个 csv 文件以存储在不可变的数据结构中。每一行都是一个入口。每个入口都有一个车站。每个车站可以有多个入口。有没有一种方法可以一次通过而不是您在下面看到的两次通过？

object NYCSubwayEntrances {
  def main(args: Array[String]) = {
    import com.github.tototoshi.csv.CSVReader
    //http://www.mta.info/developers/data/nyct/subway/StationEntrances.csv
    val file = new java.io.File("StationEntrances.csv")
    val reader = CSVReader.open(file)
    reader.readNext //consume headers
    val entranceMap = list2multimap(
      reader.all map {
        case fields: List[String] => 
          // println(fields)
          (
            fields(2), 
            Entrance(
              fields(14).toBoolean,
              Option(fields(15)),
              fields(16).toBoolean,
              fields(17),
              fields(18) match {case "YES" => true case _ => false},
              fields(19) match {case "YES" => true case _ => false},
              fields(20),
              fields(21),
              fields(22),
              fields(23),
              fields(24).toInt,
              fields(25).toInt
            )
          )
        }
      )
    reader.close
    val reader2 = CSVReader.open(file)
    reader2.readNext //consume headers
    val stations = reader2.all map { case fields: List[String] =>
      Station(
        fields(2),
        fields(0),
        fields(1),
        colate(scala.collection.immutable.ListSet[String](
          fields(3),
          fields(4),
          fields(5),
          fields(6),
          fields(7),
          fields(8),
          fields(9),
          fields(10),
          fields(11),
          fields(12),
          fields(13)
        )),
        entranceMap(fields(2)).toList
      )
    }
    reader2.close

    import net.liftweb.json._
    import net.liftweb.json.Serialization.write
    implicit val formats = Serialization.formats(NoTypeHints)
    println(pretty(render(parse(write(stations.toSet)))))
  }

  import scala.collection.mutable.{HashMap, Set, MultiMap}

  def list2multimap[A, B](list: List[(A, B)]) = 
    list.foldLeft(new HashMap[A, Set[B]] with MultiMap[A, B]){(acc, pair) => acc.addBinding(pair._1, pair._2)}

  def colate(set: scala.collection.immutable.ListSet[String]): List[String] =
    ((List[String]() ++ set) diff List("")).reverse
}

case class Station(name: String, division: String, line: String, routes: List[String], entrances: List[Entrance]) {}
case class Entrance(ada: Boolean, adaNotes: Option[String], freeCrossover: Boolean, entranceType: String, entry: Boolean, exitOnly: Boolean, entranceStaffing: String, northSouthStreet: String, eastWestStreet: String, corner: String, latitude: Integer, longitude: Integer) {}

可以在 https://github.com/AEtherSurfer/NYCSubwayEntrances找到具有所有正确依赖项的 sbt 项目

StationEntrances.csv 来自http://www.mta.info/developers/sbwy_entrance.html

4

1 回答 1

1

我有以下片段。第一个解决方案用于groupBy对与同一车站相关的入口进行分组。它不假定行已排序。虽然它只读取一次文件，但它确实执行了 3 次传递（一次读取内存中的所有内容，一次用于创建工作站，另groupBy一次用于创建工作站）。提取器的代码见最后Row。

val stations = {
  val file = new java.io.File("StationEntrances.csv")
  val reader = com.github.tototoshi.csv.CSVReader.open(file)
  val byStation = reader
    .all     // read all in memory
    .drop(1) // drop header
    .groupBy {
      case List(division, line, station, _*) => (division, line, station)
    }
  reader.close
  byStation.values.toList map { rows =>
    val entrances = rows map { case Row(_, _, _, _, entrance) => entrance }
    rows.head match {
      case Row(division, line, station, routes, _) =>
        Station(
          division, line, station,
          routes.toList.filter(_ != ""),
          entrances)
    }
  }
}

此解决方案假定行已排序并且应该更快，因为它只执行一次并在读取文件时构建结果列表。 

val stations2  = {
  import collection.mutable.ListBuffer
  def processByChunk(iter: Iterator[Seq[String]], acc: ListBuffer[Station])
           : List[Station] = {
    if (!iter.hasNext) acc.toList
    else {
      val head = iter.next
      val marker = head.take(3)
      val (rows, rest) = iter.span(_ startsWith marker)
      val entrances = (head :: rows.toList) map {
        case Row(_, _, _, _, entrance) => entrance
      }
      val station = head match {
        case Row(division, line, station, routes, _) =>
          Station(
            division, line, station,
            routes.toList.filter(_ != ""),
            entrances)
      }
      processByChunk(rest, acc += station)
    }
  }
  val file = new java.io.File("StationEntrances.csv")
  val reader = com.github.tototoshi.csv.CSVReader.open(file)
  val stations = processByChunk(reader.iterator.drop(1), ListBuffer())
  reader.close
  stations
}

我创建了一个专用提取器来获取给定线路的路线/入口。我认为它使代码更具可读性，但如果您正在处理列表，调用fields(0)tofields(25)并不是最佳的，因为每个调用都必须遍历列表。提取器避免了这种情况。对于大多数 Java csv 解析器，您通常会得到Array[String]，所以这通常不是问题。最后，csv 解析通常不会返回空字符串，因此您可能希望if (adaNotes == "") None else Some(adaNotes)使用Option(adaNotes).

object Row {
  def unapply(s: Seq[String]) = s match {
    case List(division, line, station, rest @ _*) =>
      val (routes,
        List(ada, adaNotes, freeCrossover, entranceType,
          entry, exitOnly, entranceStaffing, northSouthStreet, eastWestStreet,
          corner, latitude, longitude)) = rest splitAt 11 // 11 routes
      Some((
        division, line, station,
        routes,
        Entrance(
          ada.toBoolean, Option(adaNotes),
          freeCrossover.toBoolean, entranceType,
          entry == "YES", exitOnly == "YES",
          entranceStaffing, northSouthStreet, eastWestStreet, corner,
          latitude.toInt, longitude.toInt)))
    case _ => None
  }
}
于 2013-03-20T05:47:36.257 回答