diff --git a/bench/archive/jmh-results-20190912131522.json b/bench/archive/jmh-results-20190912131522.json new file mode 100644 index 000000000..c92a18ba3 --- /dev/null +++ b/bench/archive/jmh-results-20190912131522.json @@ -0,0 +1,108 @@ +[ + { + "jmhVersion" : "1.21", + "benchmark" : "org.locationtech.rasterframes.bench.CellTypeBench.fromRow", + "mode" : "avgt", + "threads" : 1, + "forks" : 1, + "jvm" : "/Library/Java/JavaVirtualMachines/jdk1.8.0_171.jdk/Contents/Home/jre/bin/java", + "jvmArgs" : [ + "-Xmx4g" + ], + "jdkVersion" : "1.8.0_171", + "vmName" : "Java HotSpot(TM) 64-Bit Server VM", + "vmVersion" : "25.171-b11", + "warmupIterations" : 8, + "warmupTime" : "10 s", + "warmupBatchSize" : 1, + "measurementIterations" : 5, + "measurementTime" : "10 s", + "measurementBatchSize" : 1, + "primaryMetric" : { + "score" : 2.835836726694286, + "scoreError" : 0.19270422040266105, + "scoreConfidence" : [ + 2.643132506291625, + 3.028540947096947 + ], + "scorePercentiles" : { + "0.0" : 2.795428349144724, + "50.0" : 2.833201108807787, + "90.0" : 2.918007291796378, + "95.0" : 2.918007291796378, + "99.0" : 2.918007291796378, + "99.9" : 2.918007291796378, + "99.99" : 2.918007291796378, + "99.999" : 2.918007291796378, + "99.9999" : 2.918007291796378, + "100.0" : 2.918007291796378 + }, + "scoreUnit" : "us/op", + "rawData" : [ + [ + 2.795428349144724, + 2.79552918770274, + 2.837017696019802, + 2.918007291796378, + 2.833201108807787 + ] + ] + }, + "secondaryMetrics" : { + } + }, + { + "jmhVersion" : "1.21", + "benchmark" : "org.locationtech.rasterframes.bench.CellTypeBench.intoRow", + "mode" : "avgt", + "threads" : 1, + "forks" : 1, + "jvm" : "/Library/Java/JavaVirtualMachines/jdk1.8.0_171.jdk/Contents/Home/jre/bin/java", + "jvmArgs" : [ + "-Xmx4g" + ], + "jdkVersion" : "1.8.0_171", + "vmName" : "Java HotSpot(TM) 64-Bit Server VM", + "vmVersion" : "25.171-b11", + "warmupIterations" : 8, + "warmupTime" : "10 s", + "warmupBatchSize" : 1, + "measurementIterations" : 5, + "measurementTime" : "10 s", + "measurementBatchSize" : 1, + "primaryMetric" : { + "score" : 0.35239272354175755, + "scoreError" : 0.007379508933042929, + "scoreConfidence" : [ + 0.3450132146087146, + 0.3597722324748005 + ], + "scorePercentiles" : { + "0.0" : 0.34961161284617265, + "50.0" : 0.3525031701325926, + "90.0" : 0.3550145438728285, + "95.0" : 0.3550145438728285, + "99.0" : 0.3550145438728285, + "99.9" : 0.3550145438728285, + "99.99" : 0.3550145438728285, + "99.999" : 0.3550145438728285, + "99.9999" : 0.3550145438728285, + "100.0" : 0.3550145438728285 + }, + "scoreUnit" : "us/op", + "rawData" : [ + [ + 0.3550145438728285, + 0.3522314263912252, + 0.3525031701325926, + 0.34961161284617265, + 0.3526028644659687 + ] + ] + }, + "secondaryMetrics" : { + } + } +] + + diff --git a/bench/archive/jmh-results-20190912133231.json b/bench/archive/jmh-results-20190912133231.json new file mode 100644 index 000000000..e73e25e85 --- /dev/null +++ b/bench/archive/jmh-results-20190912133231.json @@ -0,0 +1,108 @@ +[ + { + "jmhVersion" : "1.21", + "benchmark" : "org.locationtech.rasterframes.bench.CellTypeBench.fromRow", + "mode" : "avgt", + "threads" : 1, + "forks" : 1, + "jvm" : "/Library/Java/JavaVirtualMachines/jdk1.8.0_171.jdk/Contents/Home/jre/bin/java", + "jvmArgs" : [ + "-Xmx4g" + ], + "jdkVersion" : "1.8.0_171", + "vmName" : "Java HotSpot(TM) 64-Bit Server VM", + "vmVersion" : "25.171-b11", + "warmupIterations" : 8, + "warmupTime" : "10 s", + "warmupBatchSize" : 1, + "measurementIterations" : 5, + "measurementTime" : "10 s", + "measurementBatchSize" : 1, + "primaryMetric" : { + "score" : 0.09014145135858256, + "scoreError" : 0.002394012003996672, + "scoreConfidence" : [ + 0.0877474393545859, + 0.09253546336257923 + ], + "scorePercentiles" : { + "0.0" : 0.08936698388038906, + "50.0" : 0.08989743516282808, + "90.0" : 0.09083590280173164, + "95.0" : 0.09083590280173164, + "99.0" : 0.09083590280173164, + "99.9" : 0.09083590280173164, + "99.99" : 0.09083590280173164, + "99.999" : 0.09083590280173164, + "99.9999" : 0.09083590280173164, + "100.0" : 0.09083590280173164 + }, + "scoreUnit" : "us/op", + "rawData" : [ + [ + 0.08989743516282808, + 0.09072300570225225, + 0.08988392924571173, + 0.09083590280173164, + 0.08936698388038906 + ] + ] + }, + "secondaryMetrics" : { + } + }, + { + "jmhVersion" : "1.21", + "benchmark" : "org.locationtech.rasterframes.bench.CellTypeBench.intoRow", + "mode" : "avgt", + "threads" : 1, + "forks" : 1, + "jvm" : "/Library/Java/JavaVirtualMachines/jdk1.8.0_171.jdk/Contents/Home/jre/bin/java", + "jvmArgs" : [ + "-Xmx4g" + ], + "jdkVersion" : "1.8.0_171", + "vmName" : "Java HotSpot(TM) 64-Bit Server VM", + "vmVersion" : "25.171-b11", + "warmupIterations" : 8, + "warmupTime" : "10 s", + "warmupBatchSize" : 1, + "measurementIterations" : 5, + "measurementTime" : "10 s", + "measurementBatchSize" : 1, + "primaryMetric" : { + "score" : 0.0804223243178353, + "scoreError" : 0.003232629425106684, + "scoreConfidence" : [ + 0.07718969489272862, + 0.08365495374294199 + ], + "scorePercentiles" : { + "0.0" : 0.07979611499160112, + "50.0" : 0.08015109899040546, + "90.0" : 0.08189719727590557, + "95.0" : 0.08189719727590557, + "99.0" : 0.08189719727590557, + "99.9" : 0.08189719727590557, + "99.99" : 0.08189719727590557, + "99.999" : 0.08189719727590557, + "99.9999" : 0.08189719727590557, + "100.0" : 0.08189719727590557 + }, + "scoreUnit" : "us/op", + "rawData" : [ + [ + 0.08189719727590557, + 0.08020909063591224, + 0.07979611499160112, + 0.08005811969535213, + 0.08015109899040546 + ] + ] + }, + "secondaryMetrics" : { + } + } +] + + diff --git a/bench/src/main/scala/org/locationtech/rasterframes/bench/CellTypeBench.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/CellTypeBench.scala new file mode 100644 index 000000000..dfc88f855 --- /dev/null +++ b/bench/src/main/scala/org/locationtech/rasterframes/bench/CellTypeBench.scala @@ -0,0 +1,52 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2019 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.bench +import java.util.concurrent.TimeUnit + +import geotrellis.raster.{CellType, DoubleUserDefinedNoDataCellType, IntUserDefinedNoDataCellType} +import org.apache.spark.sql.catalyst.InternalRow +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.openjdk.jmh.annotations._ + +@BenchmarkMode(Array(Mode.AverageTime)) +@State(Scope.Benchmark) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +class CellTypeBench { + var row: InternalRow = _ + var ct: CellType = _ + @Setup(Level.Trial) + def setupData(): Unit = { + ct = IntUserDefinedNoDataCellType(scala.util.Random.nextInt()) + val o: CellType = DoubleUserDefinedNoDataCellType(scala.util.Random.nextDouble()) + row = o.toInternalRow + } + + @Benchmark + def fromRow(): CellType = { + row.to[CellType] + } + + @Benchmark + def intoRow(): InternalRow = { + ct.toInternalRow + } +} diff --git a/bench/src/main/scala/org/locationtech/rasterframes/bench/TileExplodeBench.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/TileExplodeBench.scala index 7f3352f69..4ece4cc98 100644 --- a/bench/src/main/scala/org/locationtech/rasterframes/bench/TileExplodeBench.scala +++ b/bench/src/main/scala/org/locationtech/rasterframes/bench/TileExplodeBench.scala @@ -22,11 +22,12 @@ package org.locationtech.rasterframes.bench import java.util.concurrent.TimeUnit +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.BoundReference +import org.apache.spark.sql.rf.TileUDT import org.locationtech.rasterframes._ -import org.apache.spark.sql._ -import org.apache.spark.sql.functions._ +import org.locationtech.rasterframes.expressions.generators.ExplodeTiles import org.openjdk.jmh.annotations._ - /** * * @author sfitch @@ -36,33 +37,32 @@ import org.openjdk.jmh.annotations._ @State(Scope.Benchmark) @OutputTimeUnit(TimeUnit.MILLISECONDS) class TileExplodeBench extends SparkEnv { - import spark.implicits._ - @Param(Array("uint8", "uint16ud255", "float32", "float64")) + //@Param(Array("uint8", "uint16ud255", "float32", "float64")) + @Param(Array("uint16ud255")) var cellTypeName: String = _ @Param(Array("256")) var tileSize: Int = _ - @Param(Array("100")) + @Param(Array("2000")) var numTiles: Int = _ @transient - var tiles: DataFrame = _ + var tiles: Array[InternalRow] = _ + + var exploder: ExplodeTiles = _ @Setup(Level.Trial) def setupData(): Unit = { - tiles = Seq.fill(numTiles)(randomTile(tileSize, tileSize, cellTypeName)) - .toDF("tile").repartition(10) - } - - @Benchmark - def arrayExplode() = { - tiles.select(posexplode(rf_tile_to_array_double($"tile"))).count() + tiles = Array.fill(numTiles)(randomTile(tileSize, tileSize, cellTypeName)) + .map(t => InternalRow(TileUDT.tileSerializer.toInternalRow(t))) + val expr = BoundReference(0, TileType, true) + exploder = new ExplodeTiles(1.0, None, Seq(expr)) } - @Benchmark def tileExplode() = { - tiles.select(rf_explode_tiles($"tile")).count() + for(t <- tiles) + exploder.eval(t) } } diff --git a/core/src/main/scala/org/apache/spark/sql/rf/RasterSourceUDT.scala b/core/src/main/scala/org/apache/spark/sql/rf/RasterSourceUDT.scala index 772bde6fe..51d204b58 100644 --- a/core/src/main/scala/org/apache/spark/sql/rf/RasterSourceUDT.scala +++ b/core/src/main/scala/org/apache/spark/sql/rf/RasterSourceUDT.scala @@ -73,7 +73,7 @@ object RasterSourceUDT { implicit val rasterSourceSerializer: CatalystSerializer[RasterSource] = new CatalystSerializer[RasterSource] { - override def schema: StructType = StructType(Seq( + override val schema: StructType = StructType(Seq( StructField("raster_source_kryo", BinaryType, false) )) diff --git a/core/src/main/scala/org/apache/spark/sql/rf/TileUDT.scala b/core/src/main/scala/org/apache/spark/sql/rf/TileUDT.scala index 66c0d98a1..094a07cf0 100644 --- a/core/src/main/scala/org/apache/spark/sql/rf/TileUDT.scala +++ b/core/src/main/scala/org/apache/spark/sql/rf/TileUDT.scala @@ -74,7 +74,7 @@ case object TileUDT { implicit def tileSerializer: CatalystSerializer[Tile] = new CatalystSerializer[Tile] { - override def schema: StructType = StructType(Seq( + override val schema: StructType = StructType(Seq( StructField("cell_context", schemaOf[TileDataContext], false), StructField("cell_data", schemaOf[Cells], false) )) diff --git a/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardSerializers.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardSerializers.scala index caf1965cc..1983f8bb9 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardSerializers.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardSerializers.scala @@ -21,6 +21,7 @@ package org.locationtech.rasterframes.encoders +import com.github.blemale.scaffeine.Scaffeine import geotrellis.proj4.CRS import geotrellis.raster._ import geotrellis.spark._ @@ -36,7 +37,7 @@ import org.locationtech.rasterframes.model.LazyCRS trait StandardSerializers { implicit val envelopeSerializer: CatalystSerializer[Envelope] = new CatalystSerializer[Envelope] { - override def schema: StructType = StructType(Seq( + override val schema: StructType = StructType(Seq( StructField("minX", DoubleType, false), StructField("maxX", DoubleType, false), StructField("minY", DoubleType, false), @@ -53,7 +54,7 @@ trait StandardSerializers { } implicit val extentSerializer: CatalystSerializer[Extent] = new CatalystSerializer[Extent] { - override def schema: StructType = StructType(Seq( + override val schema: StructType = StructType(Seq( StructField("xmin", DoubleType, false), StructField("ymin", DoubleType, false), StructField("xmax", DoubleType, false), @@ -71,7 +72,7 @@ trait StandardSerializers { } implicit val gridBoundsSerializer: CatalystSerializer[GridBounds] = new CatalystSerializer[GridBounds] { - override def schema: StructType = StructType(Seq( + override val schema: StructType = StructType(Seq( StructField("colMin", IntegerType, false), StructField("rowMin", IntegerType, false), StructField("colMax", IntegerType, false), @@ -91,7 +92,7 @@ trait StandardSerializers { } implicit val crsSerializer: CatalystSerializer[CRS] = new CatalystSerializer[CRS] { - override def schema: StructType = StructType(Seq( + override val schema: StructType = StructType(Seq( StructField("crsProj4", StringType, false) )) override def to[R](t: CRS, io: CatalystIO[R]): R = io.create( @@ -106,18 +107,19 @@ trait StandardSerializers { } implicit val cellTypeSerializer: CatalystSerializer[CellType] = new CatalystSerializer[CellType] { - override def schema: StructType = StructType(Seq( + import StandardSerializers._ + override val schema: StructType = StructType(Seq( StructField("cellTypeName", StringType, false) )) override def to[R](t: CellType, io: CatalystIO[R]): R = io.create( - io.encode(t.toString()) + io.encode(ct2sCache.get(t)) ) override def from[R](row: R, io: CatalystIO[R]): CellType = - CellType.fromName(io.getString(row, 0)) + s2ctCache.get(io.getString(row, 0)) } implicit val projectedExtentSerializer: CatalystSerializer[ProjectedExtent] = new CatalystSerializer[ProjectedExtent] { - override def schema: StructType = StructType(Seq( + override val schema: StructType = StructType(Seq( StructField("extent", schemaOf[Extent], false), StructField("crs", schemaOf[CRS], false) )) @@ -134,7 +136,7 @@ trait StandardSerializers { } implicit val spatialKeySerializer: CatalystSerializer[SpatialKey] = new CatalystSerializer[SpatialKey] { - override def schema: StructType = StructType(Seq( + override val schema: StructType = StructType(Seq( StructField("col", IntegerType, false), StructField("row", IntegerType, false) )) @@ -151,7 +153,7 @@ trait StandardSerializers { } implicit val spacetimeKeySerializer: CatalystSerializer[SpaceTimeKey] = new CatalystSerializer[SpaceTimeKey] { - override def schema: StructType = StructType(Seq( + override val schema: StructType = StructType(Seq( StructField("col", IntegerType, false), StructField("row", IntegerType, false), StructField("instant", LongType, false) @@ -171,7 +173,7 @@ trait StandardSerializers { } implicit val cellSizeSerializer: CatalystSerializer[CellSize] = new CatalystSerializer[CellSize] { - override def schema: StructType = StructType(Seq( + override val schema: StructType = StructType(Seq( StructField("width", DoubleType, false), StructField("height", DoubleType, false) )) @@ -188,7 +190,7 @@ trait StandardSerializers { } implicit val tileLayoutSerializer: CatalystSerializer[TileLayout] = new CatalystSerializer[TileLayout] { - override def schema: StructType = StructType(Seq( + override val schema: StructType = StructType(Seq( StructField("layoutCols", IntegerType, false), StructField("layoutRows", IntegerType, false), StructField("tileCols", IntegerType, false), @@ -211,7 +213,7 @@ trait StandardSerializers { } implicit val layoutDefinitionSerializer = new CatalystSerializer[LayoutDefinition] { - override def schema: StructType = StructType(Seq( + override val schema: StructType = StructType(Seq( StructField("extent", schemaOf[Extent], true), StructField("tileLayout", schemaOf[TileLayout], true) )) @@ -228,7 +230,7 @@ trait StandardSerializers { } implicit def boundsSerializer[T >: Null: CatalystSerializer]: CatalystSerializer[KeyBounds[T]] = new CatalystSerializer[KeyBounds[T]] { - override def schema: StructType = StructType(Seq( + override val schema: StructType = StructType(Seq( StructField("minKey", schemaOf[T], true), StructField("maxKey", schemaOf[T], true) )) @@ -245,7 +247,7 @@ trait StandardSerializers { } def tileLayerMetadataSerializer[T >: Null: CatalystSerializer]: CatalystSerializer[TileLayerMetadata[T]] = new CatalystSerializer[TileLayerMetadata[T]] { - override def schema: StructType = StructType(Seq( + override val schema: StructType = StructType(Seq( StructField("cellType", schemaOf[CellType], false), StructField("layout", schemaOf[LayoutDefinition], false), StructField("extent", schemaOf[Extent], false), @@ -273,7 +275,7 @@ trait StandardSerializers { implicit def rasterSerializer: CatalystSerializer[Raster[Tile]] = new CatalystSerializer[Raster[Tile]] { import org.apache.spark.sql.rf.TileUDT.tileSerializer - override def schema: StructType = StructType(Seq( + override val schema: StructType = StructType(Seq( StructField("tile", TileType, false), StructField("extent", schemaOf[Extent], false) )) @@ -293,3 +295,12 @@ trait StandardSerializers { implicit val spaceTimeKeyTLMSerializer = tileLayerMetadataSerializer[SpaceTimeKey] } + +object StandardSerializers { + private val s2ctCache = Scaffeine().build[String, CellType]( + (s: String) => CellType.fromName(s) + ) + private val ct2sCache = Scaffeine().build[CellType, String]( + (ct: CellType) => ct.toString() + ) +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/ProjectedLayerMetadataAggregate.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/ProjectedLayerMetadataAggregate.scala index 0f1b4727a..f181d0eef 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/ProjectedLayerMetadataAggregate.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/ProjectedLayerMetadataAggregate.scala @@ -118,7 +118,7 @@ object ProjectedLayerMetadataAggregate { private[expressions] object InputRecord { implicit val serializer: CatalystSerializer[InputRecord] = new CatalystSerializer[InputRecord]{ - override def schema: StructType = StructType(Seq( + override val schema: StructType = StructType(Seq( StructField("extent", CatalystSerializer[Extent].schema, false), StructField("crs", CatalystSerializer[CRS].schema, false), StructField("cellType", CatalystSerializer[CellType].schema, false), @@ -147,7 +147,7 @@ object ProjectedLayerMetadataAggregate { } def write(buffer: MutableAggregationBuffer): Unit = { - val encoded = (this).toRow + val encoded = this.toRow for(i <- 0 until encoded.size) { buffer(i) = encoded(i) } @@ -157,7 +157,7 @@ object ProjectedLayerMetadataAggregate { private[expressions] object BufferRecord { implicit val serializer: CatalystSerializer[BufferRecord] = new CatalystSerializer[BufferRecord] { - override def schema: StructType = StructType(Seq( + override val schema: StructType = StructType(Seq( StructField("extent", CatalystSerializer[Extent].schema, true), StructField("cellType", CatalystSerializer[CellType].schema, true), StructField("cellSize", CatalystSerializer[CellSize].schema, true) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/ExplodeTiles.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/ExplodeTiles.scala index 2a70be585..74589c0ae 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/ExplodeTiles.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/ExplodeTiles.scala @@ -24,8 +24,8 @@ package org.locationtech.rasterframes.expressions.generators import geotrellis.raster._ import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback -import org.apache.spark.sql.catalyst.expressions.{Expression, Generator, GenericInternalRow} +import org.apache.spark.sql.catalyst.expressions.codegen.{BufferHolder, CodegenFallback, UnsafeRowWriter} +import org.apache.spark.sql.catalyst.expressions.{Expression, Generator, GenericInternalRow, UnsafeRow} import org.apache.spark.sql.types._ import org.locationtech.rasterframes._ import org.locationtech.rasterframes.expressions.DynamicExtractors @@ -87,14 +87,17 @@ case class ExplodeTiles( cfor(0)(_ < rows, _ + 1) { row => cfor(0)(_ < cols, _ + 1) { col => val rowIndex = row * cols + col - val outCols = Array.ofDim[Any](numOutCols) - outCols(0) = col - outCols(1) = row + val outRow = new UnsafeRow(numOutCols) + val buffer = new BufferHolder(outRow) + val writer = new UnsafeRowWriter(buffer, numOutCols) + writer.write(0, col) + writer.write(1, row) cfor(0)(_ < tiles.length, _ + 1) { index => val tile = tiles(index) - outCols(index + 2) = if(tile == null) doubleNODATA else tile.getDouble(col, row) + val cell: Double = if (tile == null) doubleNODATA else tile.getDouble(col, row) + writer.write(index + 2, cell) } - retval(rowIndex) = new GenericInternalRow(outCols) + retval(rowIndex) = outRow } } if(sampleFraction > 0.0 && sampleFraction < 1.0) sample(retval) diff --git a/core/src/main/scala/org/locationtech/rasterframes/model/CellContext.scala b/core/src/main/scala/org/locationtech/rasterframes/model/CellContext.scala index 95a2e1bf0..dfc083774 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/model/CellContext.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/model/CellContext.scala @@ -29,7 +29,7 @@ import CatalystSerializer._ case class CellContext(tileContext: TileContext, tileDataContext: TileDataContext, colIndex: Short, rowIndex: Short) object CellContext { implicit val serializer: CatalystSerializer[CellContext] = new CatalystSerializer[CellContext] { - override def schema: StructType = StructType(Seq( + override val schema: StructType = StructType(Seq( StructField("tileContext", schemaOf[TileContext], false), StructField("tileDataContext", schemaOf[TileDataContext], false), StructField("colIndex", ShortType, false), diff --git a/core/src/main/scala/org/locationtech/rasterframes/model/Cells.scala b/core/src/main/scala/org/locationtech/rasterframes/model/Cells.scala index 1f7ae4d75..3a54446e1 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/model/Cells.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/model/Cells.scala @@ -68,7 +68,7 @@ object Cells { } implicit def cellsSerializer: CatalystSerializer[Cells] = new CatalystSerializer[Cells] { - override def schema: StructType = + override val schema: StructType = StructType( Seq( StructField("cells", BinaryType, true), diff --git a/core/src/main/scala/org/locationtech/rasterframes/model/TileContext.scala b/core/src/main/scala/org/locationtech/rasterframes/model/TileContext.scala index c848e9121..436b46982 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/model/TileContext.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/model/TileContext.scala @@ -41,7 +41,7 @@ object TileContext { case _ => None } implicit val serializer: CatalystSerializer[TileContext] = new CatalystSerializer[TileContext] { - override def schema: StructType = StructType(Seq( + override val schema: StructType = StructType(Seq( StructField("extent", schemaOf[Extent], false), StructField("crs", schemaOf[CRS], false) )) diff --git a/core/src/main/scala/org/locationtech/rasterframes/model/TileDataContext.scala b/core/src/main/scala/org/locationtech/rasterframes/model/TileDataContext.scala index 9f6bd358f..addc4aee5 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/model/TileDataContext.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/model/TileDataContext.scala @@ -41,7 +41,7 @@ object TileDataContext { } implicit val serializer: CatalystSerializer[TileDataContext] = new CatalystSerializer[TileDataContext] { - override def schema: StructType = StructType(Seq( + override val schema: StructType = StructType(Seq( StructField("cellType", schemaOf[CellType], false), StructField("dimensions", schemaOf[TileDimensions], false) )) diff --git a/core/src/main/scala/org/locationtech/rasterframes/model/TileDimensions.scala b/core/src/main/scala/org/locationtech/rasterframes/model/TileDimensions.scala index e419ac668..fbbdfebf1 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/model/TileDimensions.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/model/TileDimensions.scala @@ -38,7 +38,7 @@ object TileDimensions { def apply(colsRows: (Int, Int)): TileDimensions = new TileDimensions(colsRows._1, colsRows._2) implicit val serializer: CatalystSerializer[TileDimensions] = new CatalystSerializer[TileDimensions] { - override def schema: StructType = StructType(Seq( + override val schema: StructType = StructType(Seq( StructField("cols", ShortType, false), StructField("rows", ShortType, false) )) diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala index 9cd259456..a5255e663 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala @@ -76,7 +76,7 @@ object RasterRef extends LazyLogging { implicit val rasterRefSerializer: CatalystSerializer[RasterRef] = new CatalystSerializer[RasterRef] { val rsType = new RasterSourceUDT() - override def schema: StructType = StructType(Seq( + override val schema: StructType = StructType(Seq( StructField("source", rsType.sqlType, false), StructField("bandIndex", IntegerType, false), StructField("subextent", schemaOf[Extent], true), diff --git a/core/src/main/scala/org/locationtech/rasterframes/tiles/ProjectedRasterTile.scala b/core/src/main/scala/org/locationtech/rasterframes/tiles/ProjectedRasterTile.scala index 6007ce3f9..235e1b851 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/tiles/ProjectedRasterTile.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/tiles/ProjectedRasterTile.scala @@ -71,7 +71,7 @@ object ProjectedRasterTile { } } implicit val serializer: CatalystSerializer[ProjectedRasterTile] = new CatalystSerializer[ProjectedRasterTile] { - override def schema: StructType = StructType(Seq( + override val schema: StructType = StructType(Seq( StructField("tile_context", schemaOf[TileContext], false), StructField("tile", TileType, false)) ) diff --git a/docs/src/main/paradox/release-notes.md b/docs/src/main/paradox/release-notes.md index 9f500b5eb..d9136d5ef 100644 --- a/docs/src/main/paradox/release-notes.md +++ b/docs/src/main/paradox/release-notes.md @@ -4,6 +4,7 @@ ### 0.8.2 +* Fixed SparkML memory pressure issue caused by unnecessary reevaluation, overallocation, and primitive boxing. ([#343](https://github.com/locationtech/rasterframes/issues/343)) * Fixed Parquet serialization issue with `RasterRef`s ([#338](https://github.com/locationtech/rasterframes/issues/338)) * Fixed `TileExploder`, `rf_agg_local_mean` and `TileColumnSupport` to support `proj_raster` struct ([#287](https://github.com/locationtech/rasterframes/issues/287), [#163](https://github.com/locationtech/rasterframes/issues/163), [#333](https://github.com/locationtech/rasterframes/issues/333)). * Various documentation improvements. diff --git a/project/plugins.sbt b/project/plugins.sbt index 502ede0e3..a6d8dcdc5 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -8,7 +8,7 @@ addSbtPlugin("com.typesafe.sbt" % "sbt-ghpages" % "0.6.2") addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.3.2") addSbtPlugin("com.lightbend.paradox" % "sbt-paradox" % "0.5.5") addSbtPlugin("io.github.jonas" % "sbt-paradox-material-theme" % "0.6.0") -addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.3.6") +addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.3.3") addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "2.1") addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.1.1") addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.1")