From 8305ba4d59d6390cd0d1e4a3e769309d686e80a2 Mon Sep 17 00:00:00 2001 From: Rijn Buve Date: Sat, 4 Jul 2015 14:53:00 +0200 Subject: [PATCH 1/2] Using all cores for long running unit tests to speed up --- pom.xml | 6 +- src/site/apt/ReleaseNotes.apt.vm | 4 + .../java/com/mapcode/EncodeDecodeTest.java | 125 +++++----- .../java/com/mapcode/ReferenceFileTest.java | 213 ++++++++++-------- 4 files changed, 185 insertions(+), 163 deletions(-) diff --git a/pom.xml b/pom.xml index fcd11b1..ea5c33a 100644 --- a/pom.xml +++ b/pom.xml @@ -68,7 +68,7 @@ UTF-8 UTF-8 - 1.7 + 1.8 2.3.1 3.0.0 @@ -160,7 +160,6 @@ - log4j log4j @@ -179,7 +178,6 @@ ${slf4j.version} - junit junit @@ -187,7 +185,6 @@ test - com.google.code.findbugs jsr305 @@ -197,7 +194,6 @@ true - com.google.code.gson gson diff --git a/src/site/apt/ReleaseNotes.apt.vm b/src/site/apt/ReleaseNotes.apt.vm index 6d00417..87b3f1f 100755 --- a/src/site/apt/ReleaseNotes.apt.vm +++ b/src/site/apt/ReleaseNotes.apt.vm @@ -9,6 +9,10 @@ Release Notes (Version ${project.version}) In any case, never depend on them for your own non-<<>> releases. #end +* 2.0.1 + + * Use multi-threading for long running test to speed them up (uses all CPU cores now). + * 2.0.0 * Fixes to the data rectangles (primarily intended for ISO proposal). diff --git a/src/test/java/com/mapcode/EncodeDecodeTest.java b/src/test/java/com/mapcode/EncodeDecodeTest.java index e00d9cc..6efdd35 100644 --- a/src/test/java/com/mapcode/EncodeDecodeTest.java +++ b/src/test/java/com/mapcode/EncodeDecodeTest.java @@ -24,6 +24,10 @@ import java.util.List; import java.util.Random; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -37,6 +41,7 @@ public class EncodeDecodeTest { private static final int NUMBER_OF_POINTS = 5000; private static final int LOG_LINE_EVERY = 500; + @Test public void encodeDecodeTestFixedSeed() throws Exception { final long seed = 1431977987367L; @@ -51,13 +56,19 @@ public void encodeDecodeTestRandomSeed() throws Exception { doEncodeDecode(seed); } - private static void doEncodeDecode(final long seed) throws UnknownMapcodeException { + private static void doEncodeDecode(final long seed) throws InterruptedException { + + // Keep error count and create thread pool. + final AtomicInteger errors = new AtomicInteger(0); + final int threads = Runtime.getRuntime().availableProcessors(); + LOG.info("encodeDecodeTest: Starting {} threads...", threads); + final ExecutorService executor = Executors.newFixedThreadPool(threads); + final Random randomGenerator = new Random(seed); - double maxDistancePrecision0Meters = 0.0; - double maxDistancePrecision1Meters = 0.0; - double maxDistancePrecision2Meters = 0.0; for (int i = 0; i < NUMBER_OF_POINTS; i++) { - boolean showLogLine = ((i % LOG_LINE_EVERY) == 0); + if ((i % LOG_LINE_EVERY) == 0) { + LOG.info("encodeDecodeTest: #{}/{}", i, NUMBER_OF_POINTS); + } // Encode location. final Point encode = Point.fromUniformlyDistributedRandomPoints(randomGenerator); @@ -73,63 +84,61 @@ private static void doEncodeDecode(final long seed) throws UnknownMapcodeExcepti assertEquals("encodeToInternational failed, result=" + resultsAll, resultsAll.get(resultsAll.size() - 1), mapcodeInternational); - // Every point must have a Mapcode. - boolean found = false; - // Walk through the list in reverse order to get International first. for (final Territory territory : Territory.values()) { - final List resultsLimited = MapcodeCodec.encode(latDeg, lonDeg, territory); - for (final Mapcode mapcode : resultsLimited) { - found = true; - - // Check if the territory matches. - assertEquals(territory, mapcode.getTerritory()); - - // Check max distance. - final String codePrecision0 = mapcode.getCode(0); - final String codePrecision1 = mapcode.getCode(1); - final String codePrecision2 = mapcode.getCode(2); - - final Point decodeLocationPrecision0 = MapcodeCodec.decode(codePrecision0, territory); - final Point decodeLocationPrecision1 = MapcodeCodec.decode(codePrecision1, territory); - final Point decodeLocationPrecision2 = MapcodeCodec.decode(codePrecision2, territory); - - final double distancePrecision0Meters = Point.distanceInMeters(encode, decodeLocationPrecision0); - final double distancePrecision1Meters = Point.distanceInMeters(encode, decodeLocationPrecision1); - final double distancePrecision2Meters = Point.distanceInMeters(encode, decodeLocationPrecision2); - - maxDistancePrecision0Meters = Math.max(maxDistancePrecision0Meters, distancePrecision0Meters); - maxDistancePrecision1Meters = Math.max(maxDistancePrecision1Meters, distancePrecision1Meters); - maxDistancePrecision2Meters = Math.max(maxDistancePrecision2Meters, distancePrecision2Meters); - - assertTrue(mapcode + " distancePrecision0Meters=" + distancePrecision0Meters + " >= " + Mapcode.getSafeMaxOffsetInMeters(0), - distancePrecision0Meters < Mapcode.getSafeMaxOffsetInMeters(0)); - assertTrue(mapcode + "distancePrecision1Meters=" + distancePrecision1Meters + " >= " + Mapcode.getSafeMaxOffsetInMeters(1), - distancePrecision1Meters < Mapcode.getSafeMaxOffsetInMeters(1)); - assertTrue(mapcode + "distancePrecision2Meters=" + distancePrecision2Meters + " >= " + Mapcode.getSafeMaxOffsetInMeters(2), - distancePrecision2Meters < Mapcode.getSafeMaxOffsetInMeters(2)); - - // Check conversion from/to alphabets. - for (final Alphabet alphabet : Alphabet.values()) { - final String mapcodeAlphabet = mapcode.getCode(alphabet); - final String mapcodeAscii = Mapcode.convertStringToPlainAscii(mapcodeAlphabet); - assertEquals(mapcode + " alphabet=" + alphabet + ", original=" + codePrecision0 + - ", mapcodeAlphabet=" + mapcodeAlphabet + ", mapcodeAscii=" + mapcodeAscii, - codePrecision0, mapcodeAscii); - } - - if (showLogLine) { - LOG.info("encodeDecodeTest: #{}/{}, result={}, mapcode={}, territory={} --> " + - "lat={}, lon={}; delta={}", i, NUMBER_OF_POINTS, - mapcode, codePrecision0, territory.getFullName(), decodeLocationPrecision0.getLatDeg(), - decodeLocationPrecision0.getLonDeg(), distancePrecision0Meters); + executor.execute(() -> { + try { + final List resultsLimited = MapcodeCodec.encode(latDeg, lonDeg, territory); + for (final Mapcode mapcode : resultsLimited) { + + // Check if the territory matches. + assertEquals(territory, mapcode.getTerritory()); + + // Check max distance. + final String codePrecision0 = mapcode.getCode(0); + final String codePrecision1 = mapcode.getCode(1); + final String codePrecision2 = mapcode.getCode(2); + + final Point decodeLocationPrecision0 = MapcodeCodec.decode(codePrecision0, territory); + final Point decodeLocationPrecision1 = MapcodeCodec.decode(codePrecision1, territory); + final Point decodeLocationPrecision2 = MapcodeCodec.decode(codePrecision2, territory); + + final double distancePrecision0Meters = Point.distanceInMeters(encode, decodeLocationPrecision0); + final double distancePrecision1Meters = Point.distanceInMeters(encode, decodeLocationPrecision1); + final double distancePrecision2Meters = Point.distanceInMeters(encode, decodeLocationPrecision2); + + if (distancePrecision0Meters >= Mapcode.getSafeMaxOffsetInMeters(0)) { + LOG.error("encodeDecodeTest: " + mapcode + " distancePrecision0Meters = " + distancePrecision0Meters + " >= " + Mapcode.getSafeMaxOffsetInMeters(0)); + errors.getAndIncrement(); + } + if (distancePrecision1Meters >= Mapcode.getSafeMaxOffsetInMeters(1)) { + LOG.error("encodeDecodeTest: " + mapcode + " distancePrecision1Meters = " + distancePrecision1Meters + " >= " + Mapcode.getSafeMaxOffsetInMeters(1)); + errors.getAndIncrement(); + } + if (distancePrecision2Meters >= Mapcode.getSafeMaxOffsetInMeters(2)) { + LOG.error("encodeDecodeTest: " + mapcode + " distancePrecision2Meters = " + distancePrecision2Meters + " >= " + Mapcode.getSafeMaxOffsetInMeters(2)); + errors.getAndIncrement(); + } + + // Check conversion from/to alphabets. + for (final Alphabet alphabet : Alphabet.values()) { + final String mapcodeAlphabet = mapcode.getCode(alphabet); + final String mapcodeAscii = Mapcode.convertStringToPlainAscii(mapcodeAlphabet); + if (!codePrecision0.equals(mapcodeAscii)) { + LOG.error("encodeDecodeTest: " + mapcode + " alphabet=" + alphabet + ", original=" + codePrecision0 + + ", mapcodeAlphabet=" + mapcodeAlphabet + ", mapcodeAscii=" + mapcodeAscii); + } + } + } + } catch (final Exception e) { + LOG.error("encodeDecodeTest: Unexpected exception: ", e); + errors.getAndIncrement(); } - showLogLine = false; - } + }); } - assertTrue(found); } - LOG.debug("encodeDecodeTest: maximum distances, precision 0, 1, 2: {}, {}, {} meters, ", - maxDistancePrecision0Meters, maxDistancePrecision1Meters, maxDistancePrecision2Meters); + executor.shutdown(); + executor.awaitTermination(60, TimeUnit.SECONDS); + assertEquals("Found errors", 0, errors.get()); } } diff --git a/src/test/java/com/mapcode/ReferenceFileTest.java b/src/test/java/com/mapcode/ReferenceFileTest.java index 8de33f7..0bcd538 100644 --- a/src/test/java/com/mapcode/ReferenceFileTest.java +++ b/src/test/java/com/mapcode/ReferenceFileTest.java @@ -26,6 +26,11 @@ import java.io.*; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -108,8 +113,12 @@ public void checkBoundariesReferenceRecordsPrecision2() throws Exception { private static void checkFile(@Nonnull final String baseFileName) throws Exception { - int error = 0; - double maxdelta = 0; + // Reset error count. + final AtomicLong deltaNm = new AtomicLong(0); + final AtomicInteger errors = new AtomicInteger(0); + final int threads = Runtime.getRuntime().availableProcessors(); + LOG.info("checkFile: Starting {} threads...", threads); + final ExecutorService executor = Executors.newFixedThreadPool(threads); // Open data file. final ChunkedFile chunkedFile = new ChunkedFile(baseFileName); @@ -122,132 +131,136 @@ private static void checkFile(@Nonnull final String baseFileName) throws Excepti // Get next record. @Nonnull final ReferenceRec reference = getNextReferenceRecord(chunkedFile); - final boolean showLogLine = ((i % LOG_LINE_EVERY) == 0); - if (showLogLine) { + if (((i % LOG_LINE_EVERY) == 0)) { LOG.debug("checkFile: #{}, file={}", i, chunkedFile.fileName); LOG.debug("checkFile: lat/lon = {}", reference.point); LOG.debug("checkFile: expected = #{}: {}", reference.mapcodes.size(), GSON.toJson(reference.mapcodes)); } ++i; - // Encode lat/lon to series of mapcodes and check the resulting mapcodes. - final List results = MapcodeCodec.encode( - reference.point.getLatDeg(), reference.point.getLonDeg()); - if (showLogLine) { - LOG.debug("checkFile: actual = #{}: {}", results.size(), GSON.toJson(results)); - } - - // Check the number of mapcodes. - if (results.isEmpty()) { - LOG.error("checkFile: encode fails, no results found for reference={}", reference); - ++error; - } - // Check encodeToInternational. - final Mapcode resultInternational = MapcodeCodec.encodeToInternational( - reference.point.getLatDeg(), reference.point.getLonDeg()); - final Mapcode expectedInternational = results.get(results.size() - 1); - if (!resultInternational.equals(expectedInternational)) { - LOG.error("checkFile: encodeToInternational fails, expected={}, got={} for reference", - expectedInternational, resultInternational, reference); - ++error; - } + executor.execute(() -> { + // Encode lat/lon to series of mapcodes and check the resulting mapcodes. + final List results = MapcodeCodec.encode( + reference.point.getLatDeg(), reference.point.getLonDeg()); - // Check the size of the results. - if (reference.mapcodes.size() != results.size()) { - final ArrayList resultsConverted = new ArrayList<>(results.size()); - for (final Mapcode mapcode : results) { - resultsConverted.add(new MapcodeRec(mapcode.getCode(2), mapcode.getTerritory())); + // Check the number of mapcodes. + if (results.isEmpty()) { + LOG.error("checkFile: encode fails, no results found for reference={}", reference); + errors.incrementAndGet(); } - LOG.error("checkFile: Encode #{} incorrect number of results:" + - "\n lat/lon = {}" + - "\n expected = #{}: {} results," + - "\n actual = #{}: {} results\n", - i, - reference.point, - reference.mapcodes.size(), - GSON.toJson(reference.mapcodes), - results.size(), - GSON.toJson(resultsConverted)); - ++error; - } - // For every mapcode in the result set, check if it is contained in the reference set. - int precision = 0; - for (final Mapcode result : results) { - boolean found = false; - for (final MapcodeRec referenceMapcodeRec : reference.mapcodes) { - precision = (referenceMapcodeRec.mapcode.lastIndexOf('-') > 4) ? 2 : 0; + // Check encodeToInternational. + final Mapcode resultInternational = MapcodeCodec.encodeToInternational( + reference.point.getLatDeg(), reference.point.getLonDeg()); + final Mapcode expectedInternational = results.get(results.size() - 1); + if (!resultInternational.equals(expectedInternational)) { + LOG.error("checkFile: encodeToInternational fails, expected={}, got={} for reference", + expectedInternational, resultInternational, reference); + errors.incrementAndGet(); + } - if (referenceMapcodeRec.territory.equals(result.getTerritory())) { - if (referenceMapcodeRec.mapcode.equals(result.getCode(precision))) { - found = true; - break; - } + // Check the size of the results. + if (reference.mapcodes.size() != results.size()) { + final ArrayList resultsConverted = new ArrayList<>(results.size()); + for (final Mapcode mapcode : results) { + resultsConverted.add(new MapcodeRec(mapcode.getCode(2), mapcode.getTerritory())); } + LOG.error("checkFile: Incorrect number of results:" + + "\n lat/lon = {}" + + "\n expected = #{}: {} results," + + "\n actual = #{}: {} results\n", + reference.point, + reference.mapcodes.size(), + GSON.toJson(reference.mapcodes), + results.size(), + GSON.toJson(resultsConverted)); + errors.incrementAndGet(); } - if (!found) { - - // This does not fail the test, but rather produces an ERROR in the log file. - // It indicates a discrepancy in the C and Java implementations. - LOG.error("checkFile: Created '{}' at {} which is not present in the reference file!\n" + - "ref={}\n" + "new={}", - result.getCode(precision), reference.point, GSON.toJson(reference), GSON.toJson(result)); - ++error; - } - } - // For every Mapcode in the reference set, check if it is contained in the result set. - for (final MapcodeRec referenceMapcodeRec : reference.mapcodes) { - precision = (referenceMapcodeRec.mapcode.lastIndexOf('-') > 4) ? 2 : 0; - boolean found = false; + // For every mapcode in the result set, check if it is contained in the reference set. + int precision = 0; for (final Mapcode result : results) { - if (referenceMapcodeRec.territory.equals(result.getTerritory())) { - if (referenceMapcodeRec.mapcode.equals(result.getCode(precision))) { - found = true; - break; + boolean found = false; + for (final MapcodeRec referenceMapcodeRec : reference.mapcodes) { + precision = (referenceMapcodeRec.mapcode.lastIndexOf('-') > 4) ? 2 : 0; + + if (referenceMapcodeRec.territory.equals(result.getTerritory())) { + if (referenceMapcodeRec.mapcode.equals(result.getCode(precision))) { + found = true; + break; + } } } + if (!found) { + + // This does not fail the test, but rather produces an ERROR in the log file. + // It indicates a discrepancy in the C and Java implementations. + LOG.error("checkFile: Created '{}' at {} which is not present in the reference file!\n" + + "ref={}\n" + "new={}", + result.getCode(precision), reference.point, GSON.toJson(reference), GSON.toJson(result)); + errors.incrementAndGet(); + } } - if (!found) { - LOG.error("checkFile: Found '{} {}' at {} in reference file, not produced by new decoder!\n" + - "ref={}", - referenceMapcodeRec.territory, referenceMapcodeRec.mapcode, reference.point, - GSON.toJson(reference)); - ++error; + + // For every Mapcode in the reference set, check if it is contained in the result set. + for (final MapcodeRec referenceMapcodeRec : reference.mapcodes) { + precision = (referenceMapcodeRec.mapcode.lastIndexOf('-') > 4) ? 2 : 0; + boolean found = false; + for (final Mapcode result : results) { + if (referenceMapcodeRec.territory.equals(result.getTerritory())) { + if (referenceMapcodeRec.mapcode.equals(result.getCode(precision))) { + found = true; + break; + } + } + } + if (!found) { + LOG.error("checkFile: Found '{} {}' at {} in reference file, not produced by new decoder!\n" + + "ref={}", + referenceMapcodeRec.territory, referenceMapcodeRec.mapcode, reference.point, + GSON.toJson(reference)); + errors.incrementAndGet(); + } } - } - // Check distance of decoded point to reference point. - for (final MapcodeRec mapcodeRec : reference.mapcodes) { - //noinspection NestedTryStatement - try { - final Point result = MapcodeCodec.decode(mapcodeRec.mapcode, mapcodeRec.territory); - final double distanceMeters = Point.distanceInMeters(reference.point, result); - maxdelta = Math.max(maxdelta, distanceMeters); - - final double maxDeltaMeters = (mapcodeRec.mapcode.lastIndexOf('-') > 4) ? - Mapcode.getSafeMaxOffsetInMeters(2) : Mapcode.getSafeMaxOffsetInMeters(0); - if (distanceMeters > maxDeltaMeters) { - LOG.error("Mapcode {} {} was generated for point {}, but decodes to point {} " + - "which is {} meters from the original point (max is {} meters).", - mapcodeRec.territory, mapcodeRec.mapcode, reference.point, result, distanceMeters, maxDeltaMeters); - ++error; + // Check distance of decoded point to reference point. + for (final MapcodeRec mapcodeRec : reference.mapcodes) { + //noinspection NestedTryStatement + try { + final Point result = MapcodeCodec.decode(mapcodeRec.mapcode, mapcodeRec.territory); + final long distanceNm = (long) (Point.distanceInMeters(reference.point, result) * 1000000.0); + synchronized (deltaNm) { + deltaNm.set(Math.max(deltaNm.get(), distanceNm)); + } + + final long maxDeltaNm = (long) (((mapcodeRec.mapcode.lastIndexOf('-') > 4) ? + Mapcode.getSafeMaxOffsetInMeters(2) : Mapcode.getSafeMaxOffsetInMeters(0)) * 1000000.0); + if (distanceNm > maxDeltaNm) { + LOG.error("Mapcode {} {} was generated for point {}, but decodes to point {} " + + "which is {} meters from the original point (max is {} meters).", + mapcodeRec.territory, mapcodeRec.mapcode, reference.point, result, + ((double) distanceNm) / 1000000.0, ((double) maxDeltaNm) / 1000000.0); + errors.incrementAndGet(); + } + } catch (final UnknownMapcodeException unknownMapcodeException) { + LOG.error("Mapcode {} {} was generated for point {}, but cannot be decoded.", + mapcodeRec.territory, mapcodeRec.mapcode, reference.point); + errors.incrementAndGet(); } - } catch (final UnknownMapcodeException unknownMapcodeException) { - LOG.error("Mapcode {} {} was generated for point {}, but cannot be decoded.", - mapcodeRec.territory, mapcodeRec.mapcode, reference.point); - ++error; } - } + }); } } catch (final EOFException e) { // OK. } finally { chunkedFile.close(); } - LOG.debug("checkFile: Maximum delta for this testset = {}", maxdelta); - assertEquals("Found errors", 0, error); + executor.shutdown(); + executor.awaitTermination(60, TimeUnit.SECONDS); + assertEquals(0, errors.get()); + assertEquals("Found errors", 0, errors.get()); + LOG.debug("checkFile: Maximum delta for this testset = {}m", ((double) deltaNm.get()) / 1000000.0); } private static class MapcodeRec { From 2ea9a33187006993c20e328984856e0921f656f4 Mon Sep 17 00:00:00 2001 From: Rijn Buve Date: Mon, 6 Jul 2015 13:28:01 +0200 Subject: [PATCH 2/2] Changed POM version number --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ea5c33a..cb330a6 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ mapcode jar - 2.0.0 + 2.0.1-SNAPSHOT Mapcode Java Library