Commit 23adb892 authored by Steven Pisarski's avatar Steven Pisarski

Addition of new ML tuning parameters and support for RabbitMQ for sending events.

parent a6ad7f79
......@@ -64,6 +64,7 @@ object Generator extends App {
new SparkContext(config.sparkUri, config.appName, sparkConf)
else new SparkContext(sparkConf)
// TODO FIXME - Remote actors are not working
val address = if (config.remoteActorAddr == null) null else config.remoteActorAddr.address
if (!config.analyzeFirst) {
// Load from cached data
......@@ -74,10 +75,7 @@ object Generator extends App {
seedEngineFromCache(sc, eventGen, config.seedEventsUri, config.eventTimeOffset, config.useNow, seedFilter)
} else {
// Load from raw data
val analyzer =
if (config.sparkUri == null) SparkAnalyzer.analyze(config.inputDef, config.eventsUri, config.fileDelim)
else SparkAnalyzer.analyze(sc, config.inputDef, config.eventsUri, config.fileDelim)
val analyzer = SparkAnalyzer.analyze(sc, config.inputDef, config.eventsUri, config.fileDelim)
val eventGen = engineFromAnalyzer(analyzer, sendPastEvents = config.sendPastEvents,
outputDefs = config.outputDefs, numSchedulerThreads = config.numSchedulerThreads,
address, config.minActors, config.maxActors)
......
......@@ -176,7 +176,7 @@ class ScheduleAndNotify(val numThreads: Int, val startTime: Date,
import org.quartz.TriggerBuilder._
import scala.collection.JavaConversions._
import scala.collection.JavaConversions._
private[this] var logger: Logger = _
......@@ -405,7 +405,7 @@ class RabbitMqRouter(val host: String, val user: String, val pass: String, val n
override def receive = {
case msg =>
logger.debug(s"Sending $msg")
channel.basicPublish("", "hello", null, msg.toString.getBytes)
channel.basicPublish("", name, null, msg.toString.getBytes)
logger.debug(s"Sent $msg")
}
}
......
package com.cablelabs.eventgen.algorithm
import com.cablelabs.eventgen.model.Field
import org.apache.spark.mllib.classification.{NaiveBayes, ClassificationModel => SparkClassificationModel}
import org.apache.spark.mllib.classification.{ClassificationModel => SparkClassificationModel, NaiveBayes}
import org.apache.spark.mllib.linalg.DenseVector
import org.apache.spark.mllib.regression.{LabeledPoint, LinearRegressionWithSGD, RegressionModel => SparkRegressionModel}
import org.apache.spark.mllib.regression.{LabeledPoint, LinearRegressionWithSGD, RegressionModel => SparkRegressionModel, RidgeRegressionWithSGD}
import org.apache.spark.rdd.RDD
/**
......@@ -149,6 +149,8 @@ trait RegressionModel extends SupervisedModel {
/**
* Implementation of a Naive Bayes classification predictive algorithm
* @param field - the field using this algorithm
* @param trainingSet - contains the labels and associated features used to train the algorithm
* @param lambda - the algorithm's smoothing parameter
*/
class NaiveBayesModel(override val field: Field, override val trainingSet: RDD[(LabeledPoint, Any)],
......@@ -162,6 +164,7 @@ class NaiveBayesModel(override val field: Field, override val trainingSet: RDD[(
/**
* Implementation of a Linear Regression predictive algorithm
* @param field - the field using this algorithm
* @param trainingSet - contains the labels and associated features used to train the algorithm
* @param weights - the weights for each individual column used during learning - the size must = the feature set size
* @param numIterations - the number of times gradient descent will run during training
......@@ -172,7 +175,8 @@ class LinearRegressionModel(override val field: Field, override val trainingSet:
extends RegressionModel {
private val actualTrainingSet = trainingSet.map(_._1)
private[algorithm] lazy override val featuresSize = actualTrainingSet.first().features.size
private[algorithm] override val model = if (weights.isEmpty)
private[algorithm] override val model =
if (weights.isEmpty)
LinearRegressionWithSGD.train(trainingSet.map(_._1), numIterations, stepSize, 1.0)
else {
require(weights.size == featuresSize)
......@@ -180,6 +184,23 @@ class LinearRegressionModel(override val field: Field, override val trainingSet:
}
}
/**
* Implementation of a Ridge Regression predictive algorithm
* @param field - the field using this algorithm
* @param trainingSet - contains the labels and associated features used to train the algorithm
* @param numIterations - the number of times gradient descent will run during training
* @param stepSize - the size of each step taken during gradient descent
* TODO - need to really test this new prediction model
*/
class RidgeRegressionModel(override val field: Field, override val trainingSet: RDD[(LabeledPoint, Any)],
val numIterations: Int, val stepSize: Double)
extends RegressionModel {
private val actualTrainingSet = trainingSet.map(_._1)
private[algorithm] lazy override val featuresSize = actualTrainingSet.first().features.size
private[algorithm] override val model =
RidgeRegressionWithSGD.train(trainingSet.map(_._1), numIterations, stepSize, 1.0)
}
/**
* Constant model that will return the longVal for each predict() & predictRaw() method call
*/
......
......@@ -262,9 +262,14 @@ class SparkAnalyzer(val data: RDD[(String, Date, Map[String, Any])], val inputDe
case algoDef:SupervisedTraining =>
logger.info("Retrieving the training sets for each fact prediction algorithm")
data.flatMap[(LabeledPoint, Any)](p => {
val features = inputDef.factAlgoFeatures(p._3, fact)
val rawFeatures = inputDef.factAlgoFeatures(p._3, fact)
/* TODO - May want to implement some optional feature scaling and this is just one way to do so
val min = rawFeatures.min
val max = rawFeatures.max
val scaledFeatures = rawFeatures.map(x => x - min / max - min)
*/
Seq((LabeledPoint(inputDef.fieldMap.get(name).get.mlTrainingValue(p._3),
new DenseVector(features.toArray)),
new DenseVector(rawFeatures.toArray)),
fact.eventValue(p._3)))
})
case _ =>
......
......@@ -31,24 +31,22 @@ abstract class InputField(override val name: String, override val description: S
* @param denormFields - fields to extract from this field data
* @param description - the description
* @param algoDef - the algorithm that determines the event frequency
* @param factPosition - the last fact position to be used in the temporal training set
*/
abstract class Temporal(override val name: String, override val description: String = "", val denormFields: Seq[String],
val algoDef: AlgorithmDefinition, override val factPosition: Int)
val algoDef: AlgorithmDefinition)
extends InputField(name, description) with TemporalRole with AlgorithmRole {
def denormalize(event: Map[String, Any]): Seq[(String, Long)]
def denormalize(event: Map[String, Any]): Seq[(String, Double)]
}
/**
* The Date implementation of the Temporal field
* see other param descriptions in super
* @param dateFmtStr - the format of the inbound date string
* @param factPosition - the last fact index to be used in the temporal training set
*/
class DateTemporal(override val name: String, override val description: String = "",
override val denormFields: Seq[String], override val algoDef: AlgorithmDefinition,
dateFmtStr: String, override val factPosition: Int)
extends Temporal(name, description, denormFields, algoDef, factPosition) with DateRole {
dateFmtStr: String)
extends Temporal(name, description, denormFields, algoDef) with DateRole {
val dateFormat = new SimpleDateFormat(dateFmtStr)
private[this] def logger = Logger(LoggerFactory.getLogger("DateTemporal"))
......@@ -57,7 +55,7 @@ class DateTemporal(override val name: String, override val description: String =
* @param event - the event to parse
* @return - the map
*/
override def denormalize(event: Map[String, Any]): Seq[(String, Long)] = {
override def denormalize(event: Map[String, Any]): Seq[(String, Double)] = {
/* TODO FIXME
This should never occur but I have seen a few exceptions where None is being returned from the Map but very
......@@ -84,7 +82,7 @@ class DateTemporal(override val name: String, override val description: String =
val dt = new DateTime(date)
// TODO - try and make more functional
var out = Seq[(String, Long)]()
var out = Seq[(String, Double)]()
/**
* Closure to map a configured field to the associated value on the event
......@@ -93,31 +91,31 @@ class DateTemporal(override val name: String, override val description: String =
*/
def mapField(field: String) = field match {
case "timestamp" =>
out = out :+ field -> dt.getMillis
out = out :+ field -> dt.getMillis.toDouble
case "hour_of_day" =>
out = out :+ field -> dt.hourOfDay.get.toLong
out = out :+ field -> dt.hourOfDay.get.toDouble
case "day_of_month" =>
out = out :+ field -> dt.dayOfMonth.get.toLong
out = out :+ field -> dt.dayOfMonth.get.toDouble
case "day_of_week" =>
out = out :+ field -> dt.dayOfWeek.get.toLong
out = out :+ field -> dt.dayOfWeek.get.toDouble
case "day_of_year" =>
out = out :+ field -> dt.dayOfYear.get.toLong
out = out :+ field -> dt.dayOfYear.get.toDouble
case "minute_of_day" =>
out = out :+ field -> dt.minuteOfDay.get.toLong
out = out :+ field -> dt.minuteOfDay.get.toDouble
case "minute_of_hour" =>
out = out :+ field -> dt.minuteOfHour.get.toLong
out = out :+ field -> dt.minuteOfHour.get.toDouble
case "month_of_year" =>
out = out :+ field -> dt.monthOfYear.get.toLong
out = out :+ field -> dt.monthOfYear.get.toDouble
case "second_of_day" =>
out = out :+ field -> dt.secondOfDay.get.toLong
out = out :+ field -> dt.secondOfDay.get.toDouble
case "second_of_minute" =>
out = out :+ field -> dt.secondOfMinute.get.toLong
out = out :+ field -> dt.secondOfMinute.get.toDouble
case "millis_of_day" =>
out = out :+ field -> dt.millisOfDay.get.toLong
out = out :+ field -> dt.millisOfDay.get.toDouble
case "week_of_weekyear" =>
out = out :+ field -> dt.weekOfWeekyear.get.toLong
out = out :+ field -> dt.weekOfWeekyear.get.toDouble
case "year" =>
out = out :+ field -> dt.year.get.toLong
out = out :+ field -> dt.year.get.toDouble
case _ =>
// do nothing
}
......
......@@ -97,15 +97,7 @@ trait PositionalRole extends FieldRole {
/**
* The role to be extended for the temporal field
*/
trait TemporalRole extends GenerationRole {
/**
* Determines which facts will be used when creating the training set for the temporal algorithm (if necessary)
* when their position values are less than this
* @return
*/
def factPosition: Int
}
trait TemporalRole extends GenerationRole
/**
* The role to be extended for all dimension types
......
......@@ -81,6 +81,13 @@ trait SupervisedTraining extends AlgorithmDefinition {
*/
def flatten: (Seq[Double]) => Seq[Double]
/**
* Used for altering a ML feature value to keep it within some bounds (i.e. 1 > x < 1) where the key contains the
* field name and the partial function is where the algorithm resides
* @return
*/
def normalize: Map[String, (Double) => Double]
/**
* Denotes the polynomial degrees to apply to the machine learning feature set
* @return
......@@ -115,21 +122,50 @@ trait ConstantDefinition extends AlgorithmDefinition {
/**
* Attributes required for training a Linear Regression algorithm
* @param omitFields - the fields to omit from the feature set
* @param weights - the field weights
* @param flatten - the function to apply to the entire feature set
* @param normalize - the function to apply to an individual feature
* @param polyDegree - the degree of polynomial to apply to the feature set
* @param iterations - the number of gradient descent iterations used during the training phase
* @param stepSize - amount to move the point during gradient descent for each step iteration
*/
class LinearRegressionDefinition(override val omitFields: Set[String], override val weights: Map[String, Int],
override val flatten: (Seq[Double]) => Seq[Double],
override val normalize: Map[String, (Double) => Double],
override val polyDegree: Int = 1, val iterations: Int,
val stepSize: Double = .001)
extends RegressionDefinition
/**
* Attributes required for training a Ridge Regression algorithm
* @param omitFields - the fields to omit from the feature set
* @param flatten - the function to apply to the entire feature set
* @param normalize - the function to apply to an individual feature
* @param polyDegree - the degree of polynomial to apply to the feature set
* @param iterations - the number of gradient descent iterations used during the training phase
* @param stepSize - amount to move the point during gradient descent for each step iteration
*/
class RidgeRegressionDefinition(override val omitFields: Set[String],
override val flatten: (Seq[Double]) => Seq[Double],
override val normalize: Map[String, (Double) => Double],
override val polyDegree: Int = 1, val iterations: Int,
val stepSize: Double = .001)
extends RegressionDefinition {
val weights = Map[String, Int]()
}
/**
* Attributes required for training a Naive Bayes classification algorithm
* @param omitFields - the fields to omit from the feature set
* @param flatten - the function to apply to the entire feature set
* @param normalize - the function to apply to an individual feature
* @param polyDegree - the degree of polynomial to apply to the feature set
* @param lambda - the smoothing parameter used when training the predictive algoritm
*/
class NaiveBayesDefinition(override val omitFields: Set[String] = Set(),
override val flatten: (Seq[Double]) => Seq[Double],
override val normalize: Map[String, (Double) => Double],
override val polyDegree: Int = 1, val lambda: Double = .001)
extends ClassificationDefinition
......
......@@ -68,7 +68,7 @@ class OutputDefinition(val protocol: String, val host: String, val port: Int, va
require(protocol != null && protocol.nonEmpty)
require(host != null && host.nonEmpty)
require(
if (protocol != null && (protocol != "stomp" && protocol != "rabbitMq"))
if (protocol != null && protocol != "rabbitMq")
routeType != null && (routeType == "topic" || routeType == "queue")
else true)
require(
......
......@@ -3,7 +3,6 @@ temporal:
description: Temporal 1
type: date
dateFormat: MM/dd/yyyy HH:mm:ss a
factPosition: -1
denormFields:
- hour_of_day
- day_of_month
......
......@@ -3,7 +3,6 @@ temporal:
description: Temporal 1
type: date
dateFormat: MM/dd/yyyy HH:mm:ss a
factPosition: 40
denormFields:
- hour_of_day
- day_of_month
......
......@@ -3,7 +3,6 @@ temporal:
description: Temporal 1
type: date
dateFormat: MM/dd/yyyy HH:mm:ss a
factPosition: -1
denormFields:
- hour_of_day
- day_of_month
......
......@@ -3,7 +3,6 @@ temporal:
description: Temporal 1
type: date
dateFormat: MM/dd/yyyy HH:mm:ss a
factPosition: -1
denormFields:
- hour_of_day
- day_of_month
......
package com.cablelabs.eventgen
import java.io.{File, FileInputStream}
import java.text.SimpleDateFormat
import com.cablelabs.eventgen.model.{InputDefinition, OutputDefinition}
/**
* Tests to ensure that the generator works with the Amdocs POC data set.
*/
class GeneratorAmdocsTest extends EngineTester {
val dateFormat = "MM-dd-yyyy HH:mm:ss a"
val dateFormatter = new SimpleDateFormat(dateFormat)
val inputDef = InputDefinition.inputDefinition(new FileInputStream(new File("testData/amdocs/definition/amdocs-input.yaml")))
val outputDefs = OutputDefinition.outputDefinitions(
new FileInputStream(new File("testData/amdocs/definition/amdocs-out.yaml")), inputDef)
val eventUri = new File("testData/amdocs/events/training-set-unaltered.csv").toURI.toString
val delim = ','
engineTest("generate should generate a new events for each dimensionality that is properly formed") {
val lastEvents = analyzer.lastEventByDim().collect()
lastEvents.foreach(p => {
val event = engine.nextEvent(p._2)
assert(inputDef.fieldMap.size == event.size)
event.foreach(p => {
assert(inputDef.fieldMap.keySet.contains(p._1))
})
})
}
}
......@@ -9,6 +9,6 @@ import org.apache.spark.SparkContext
object JmsConsumer extends App {
new SparkContext("local[4]", "test")
val notifier = new Notifier
TestActors.activeMqConsumer("tcp", "bda-active01", 61616, "CMConnectionInfoXML", "topic", notifier)
TestActors.activeMqConsumer("tcp", "localhost", 61616, "CMConnectionInfoXML", "topic", notifier)
Thread.sleep(10000)
}
......@@ -23,7 +23,7 @@ class ScheduleAndNotifyTest extends SparkTestUtils with BeforeAndAfter {
Set[OutputField](
new IntOutput(name = "intOutput", inputField = new IntDimension(name = "intDim", position = 4)),
new DateOutput(name = "dateOutput", inputField = new DateTemporal(name = "dateTemporal",
denormFields = List[String](), algoDef = new ConstantIntDefinition(1000), factPosition = 1, dateFmtStr = dateFmtStr),
denormFields = List[String](), algoDef = new ConstantIntDefinition(1000), dateFmtStr = dateFmtStr),
dateFmtStr = dateFmtStr)),
OutputEventFormatters.jsonFormat)
......
package com.cablelabs.eventgen.algorithm
import java.io.{File, FileInputStream}
import java.text.SimpleDateFormat
import com.cablelabs.eventgen.model.InputDefinition
/**
* Testing the fact predictions from the Amdocs POC training set
*/
class AmdocsFactPredictionsTest extends FactPredictionsTester {
// TODO - Even though this is passing, need to determine why the ML algorithm is not predicting with same type of fingerprint
val dateFormat = "HH:mm:ss a"
val dateFormatter = new SimpleDateFormat(dateFormat)
val inputDef = InputDefinition.inputDefinition(
new FileInputStream(new File("testData/amdocs/definition/amdocs-input.yaml")))
// val eventUri = new File("testData/amdocs/events/training-set2.csv").toURI.toString
val eventUri = new File("testData/amdocs/events/training-set-unaltered.csv").toURI.toString
val delim = ','
outputValues = true
threshold = .75
}
package com.cablelabs.eventgen.algorithm
import com.cablelabs.eventgen.AnalyzerTester
import com.cablelabs.eventgen.model.{LinearRegressionDefinition, NaiveBayesDefinition}
import com.cablelabs.eventgen.model.{LinearRegressionDefinition, NaiveBayesDefinition, RidgeRegressionDefinition}
/**
* Superclass designed to run tests against prediction models against a known set of data
......@@ -12,22 +12,22 @@ abstract class FactPredictionsTester extends AnalyzerTester {
// minimal training set
var ignoreFacts = Set[String]()
var outputValues: Boolean = false
var threshold = .90
analyzerTest("Analyze fact preditions to ensure the average prediction is within 90% of the average label") {
analyzerTest("Analyze fact preditions to ensure the average prediction is greater than the expected threshold") {
inputDef.positionalFacts.filter(f => !ignoreFacts.contains(f.name))foreach(fact => {
val trainingSet = analyzer.factTrainingSet(fact.name)
assert(trainingSet != null && trainingSet.count() == 1000)
var trainingSet = analyzer.factTrainingSet(fact.name)
val model = fact.algoDef match {
case lrDef: LinearRegressionDefinition =>
new LinearRegressionModel(fact, analyzer.factTrainingSet(fact.name),
inputDef.algoWeights(fact), lrDef.iterations, lrDef.stepSize)
case rrDef: RidgeRegressionDefinition =>
new RidgeRegressionModel(fact, analyzer.factTrainingSet(fact.name), rrDef.iterations, rrDef.stepSize)
case nbDef: NaiveBayesDefinition =>
new NaiveBayesModel(fact, analyzer.factTrainingSet(fact.name), nbDef.lambda)
}
println(s"Validating model for fact with name - ${fact.name}")
ModelValidator.validateModel(model, trainingSet.collect(), analyzer, 0.90, outputValues = outputValues)
ModelValidator.validateModel(model, trainingSet.collect(), analyzer, threshold, outputValues = outputValues)
})
}
}
......@@ -12,7 +12,7 @@ object ModelValidator {
def validateModel(model: Model, trainingSet: Array[(LabeledPoint, Any)], analyzer: SparkAnalyzer, accuracy: Double,
outputValues: Boolean): Unit = model match {
case model: LinearRegressionModel =>
case model: RegressionModel =>
// Predict based on the actual training set
var diff = 0d
var totalPred = 0d
......@@ -35,7 +35,7 @@ object ModelValidator {
if (label < minLabel) minLabel = label
if (label > maxLabel) maxLabel = label
if (outputValues) println(s"label = $label - pred = $pred")
if (outputValues) println(s"pred = $pred - label = $label - input = ${p._1.features}")
})
val averageLabel = totalLabel / predCount
......@@ -57,6 +57,7 @@ object ModelValidator {
// Predict based on the actual events used for training
val events = analyzer.events().collect()
// val events = Random.shuffle(analyzer.events().collect().toSet)
assert(events.size == trainingSet.size)
var totalPred2 = 0d
events.foreach(event => model.field match {
......@@ -81,12 +82,8 @@ object ModelValidator {
val pred = model.predict(p._1.features.toArray)
predCount += 1
val labelValue = model.labelValue(p._1.label)
if (labelValue == pred) {
numMatches += 1
}
if (outputValues) {
println(s"label = $labelValue - pred = $pred")
}
if (labelValue == pred) numMatches += 1
if (outputValues) println(s"pred = $pred - label = $labelValue - input = ${p._1.features}")
})
val successPercentage = numMatches.toDouble / predCount.toDouble
......
......@@ -11,7 +11,8 @@ import org.apache.spark.mllib.regression.LabeledPoint
class NaiveBayesModelTest extends SparkTestUtils {
val flatten = new AlgoFlattenYAML("", 0, 0).flatten
val fact = new StringFact("foo", "", 1, new NaiveBayesDefinition(flatten = flatten, lambda = 5.0, polyDegree = 2))
val fact = new StringFact("foo", "", 1, new NaiveBayesDefinition(flatten = flatten, lambda = 5.0, polyDegree = 2,
normalize = Map[String, (Double) => Double]()))
val trainingSet: Seq[(LabeledPoint, Any)] = Seq(
(LabeledPoint(1.1, new DenseVector(Array(10.0, 20.0))), "ABC"),
(LabeledPoint(1.2, new DenseVector(Array(20.0, 30.0))), "DEF"),
......
......@@ -30,12 +30,19 @@ class SparkAnalyzerGenericTest extends AnalyzerTester {
}
analyzerTest("Fact training set should return the proper number of training sets and features for the first fact") {
val factName = inputDef.positionalFacts(0).name
val trainingSet = analyzer.factTrainingSet(factName).collect()
assert(9 == trainingSet.size)
val rawFeatures = inputDef.temporal.denormFields.size + inputDef.dimensionSet.size
val expectedNumFeatures = rawFeatures * inputDef.facts.get(factName).get.algoDef.asInstanceOf[SupervisedTraining].polyDegree
assert(expectedNumFeatures == trainingSet.iterator.next()._1.features.size)
val fact1 = inputDef.positionalFacts(0).name
val ts1 = analyzer.factTrainingSet(fact1).collect()
assert(9 == ts1.size)
val rawFeatures1 = inputDef.temporal.denormFields.size + inputDef.dimensionSet.size
val expectedNumFeatures1 = rawFeatures1 * inputDef.facts.get(fact1).get.algoDef.asInstanceOf[SupervisedTraining].polyDegree
assert(expectedNumFeatures1 == ts1.iterator.next()._1.features.size)
val fact2 = inputDef.positionalFacts(1).name
val ts2 = analyzer.factTrainingSet(fact2).collect()
assert(9 == ts2.size)
val rawFeatures2 = inputDef.temporal.denormFields.size + inputDef.dimensionSet.size + 1
val expectedNumFeatures2 = rawFeatures2 * inputDef.facts.get(fact1).get.algoDef.asInstanceOf[SupervisedTraining].polyDegree
assert(expectedNumFeatures2 == ts2.iterator.next()._1.features.size)
}
analyzerTest("The return from dimEventsCount() should be equivalent to the actual") {
......
package com.cablelabs.eventgen.model
import com.cablelabs.eventgen.UnitSpec
/**
* Tests to ensure feature normalization configuration returns the expected function
*/
class AlgoFieldNormalizeYAMLTest extends UnitSpec {
test("No normalization") {
val flattenDef = new AlgoFieldNormalizeYAML("name", "foo", 1, 1, 0)
assert(flattenDef.normalize != null)
assert(99d == flattenDef.normalize(99d))
}
test("Log base 2 x 1") {
val flattenDef = new AlgoFieldNormalizeYAML("name", "log", 2, 1, 0)
assert(flattenDef.normalize != null)
assert(applyLog(99d, 2, 1) == flattenDef.normalize(99d))
}
test("Natural Log x 1") {
val flattenDef = new AlgoFieldNormalizeYAML("name", "naturalLog", 0, 1, 0)
assert(flattenDef.normalize != null)
assert(applyNaturalLog(99d, 1) == flattenDef.normalize(99d))
}
test("Log base 5 x 3") {
val flattenDef = new AlgoFieldNormalizeYAML("name", "log", 5, 3, 0)
assert(flattenDef.normalize != null)
assert(applyLog(99d, 5, 3) == flattenDef.normalize(99d))
}
test("Root base 2 x 1") {
val flattenDef = new AlgoFieldNormalizeYAML("name", "root", 2, 1, 0)
assert(flattenDef.normalize != null)
assert(applyRoot(99d, 2, 1) == flattenDef.normalize(99d))
}
test("Root base 5 x 3") {
val flattenDef = new AlgoFieldNormalizeYAML("name", "root", 5, 3, 0)
assert(flattenDef.normalize != null)
assert(applyRoot(99d, 5, 3) == flattenDef.normalize(99d))
}
test("Divide by 0") {
val flattenDef = new AlgoFieldNormalizeYAML("name", "divide", 5, 3, 0)
assert(flattenDef.normalize != null)
assert(99d == flattenDef.normalize(99d))
}
test("Divide by 1") {
val flattenDef = new AlgoFieldNormalizeYAML("name", "divide", 5, 3, 1)
assert(flattenDef.normalize != null)
assert(99d == flattenDef.normalize(99d))
}
test("Divide by 3") {
val flattenDef = new AlgoFieldNormalizeYAML("name", "divide", 5, 3, 3)
assert(flattenDef.normalize != null)
assert(99d / 3 == flattenDef.normalize(99d))
}
def applyLog(value: Double, base: Int, iterations: Int): Double = {
def loop(value: Double, base: Int, iterations: Int): Double =
if (iterations < 1) value
else loop(math.log10(math.abs(value)) / math.log10(base) * (if (value < 0) -1 else 1), base, iterations - 1)
loop(value, base, iterations)
}
def applyNaturalLog(value: Double, iterations: Int): Double = {
def loop(value: Double, iterations: Int): Double =
if (iterations < 1) value
else loop(math.log(math.abs(value)) * (if (value < 0) -1 else 1), iterations - 1)
loop(value, iterations)