Commit 1be62cba authored by Steven Pisarski's avatar Steven Pisarski

Renamed temporal field factIndex to factPosition for clarity

parent 70b9beab
......@@ -31,10 +31,10 @@ abstract class InputField(override val name: String, override val description: S
* @param denormFields - fields to extract from this field data
* @param description - the description
* @param algoDef - the algorithm that determines the event frequency
* @param factIndex - the last fact index to be used in the temporal training set
* @param factPosition - the last fact position to be used in the temporal training set
*/
abstract class Temporal(override val name: String, override val description: String = "", val denormFields: Seq[String],
val algoDef: AlgorithmDefinition, override val factIndex: Int)
val algoDef: AlgorithmDefinition, override val factPosition: Int)
extends InputField(name, description) with TemporalRole {
def denormalize(event: Map[String, Any]): Map[String, Long]
}
......@@ -43,12 +43,12 @@ abstract class Temporal(override val name: String, override val description: Str
* The Date implementation of the Temporal field
* see other param descriptions in super
* @param dateFmtStr - the format of the inbound date string
* @param factIndex - the last fact index to be used in the temporal training set
* @param factPosition - the last fact index to be used in the temporal training set
*/
class DateTemporal(override val name: String, override val description: String = "",
override val denormFields: Seq[String], override val algoDef: AlgorithmDefinition,
dateFmtStr: String, override val factIndex: Int)
extends Temporal(name, description, denormFields, algoDef, factIndex) with DateRole {
dateFmtStr: String, override val factPosition: Int)
extends Temporal(name, description, denormFields, algoDef, factPosition) with DateRole {
val dateFormat = new SimpleDateFormat(dateFmtStr)
/**
......
......@@ -101,9 +101,10 @@ trait TemporalRole extends GenerationRole {
/**
* Determines which facts will be used when creating the training set for the temporal algorithm (if necessary)
* when their position values are less than this
* @return
*/
def factIndex: Int
def factPosition: Int
}
/**
......
......@@ -167,7 +167,7 @@ class InputDefinition(val temporal: Temporal, val dimensionSet: Set[Dimension],
def temporalAlgoFeatures(event: Map[String, Any]): Seq[Double] = {
temporal.algoDef match {
case algoDef:SupervisedTraining =>
algoFeatures(event, temporal.factIndex, algoDef)
algoFeatures(event, temporal.factPosition, algoDef)
case _ => Seq[Double]()
}
}
......@@ -201,7 +201,7 @@ class InputDefinition(val temporal: Temporal, val dimensionSet: Set[Dimension],
if (factPosition < 0) true
else p.position < factPosition).map(_.mlTrainingValue(event))
// TODO - determine if we should allow for filtering facts too
// positionalFacts.filter(p => p.position < factIndex && !algo.omitFields.contains(p.name)).map(_.mlTrainingValue(event))
// positionalFacts.filter(p => p.position < factPosition && !algo.omitFields.contains(p.name)).map(_.mlTrainingValue(event))
AlgorithmUtil.polynomial(algo.polyDegree, AlgorithmUtil.flatten(features.toSeq, algo.flatten))
}
......@@ -215,8 +215,8 @@ class InputDefinition(val temporal: Temporal, val dimensionSet: Set[Dimension],
// TODO - add these weights to the temporal definition
val weights = temporal.denormFields.map(f => 100d) ++:
positionalDims.map(f => 2000d) ++:
(if (temporal.factIndex < 0) positionalFacts.map(fact => 10d)
else positionalFacts.filter(_.position < temporal.factIndex).map(f => 10d))
(if (temporal.factPosition < 0) positionalFacts.map(fact => 10d)
else positionalFacts.filter(_.position < temporal.factPosition).map(f => 10d))
AlgorithmUtil.polynomialWeights(algoDef.polyDegree, weights)
case _ =>
Seq[Double]()
......@@ -249,14 +249,14 @@ class TemporalFieldYaml(@JsonProperty("name") name: String,
@JsonProperty("description") description: String = "",
@JsonProperty("type") fieldType: String,
@JsonProperty("dateFormat") dateFormat: String,
@JsonProperty("factIndex") factIndex: Int,
@JsonProperty("factPosition") factPosition: Int,
@JsonProperty("denormFields") jDenormFields: java.util.List[String],
@JsonProperty("algo") algo: AlgoYaml) {
require(name != null)
require(fieldType != null && fieldType == "date") // Currently only supports type Date
if (fieldType == "date") require(dateFormat != null)
val temporalField = new DateTemporal(name, description, jDenormFields.asScala, algo.algorithm, dateFormat, factIndex)
val temporalField = new DateTemporal(name, description, jDenormFields.asScala, algo.algorithm, dateFormat, factPosition)
}
/**
......
......@@ -3,7 +3,7 @@ temporal:
description: Temporal 1
type: date
dateFormat: MM/dd/yyyy HH:mm:ss a
factIndex: -1
factPosition: -1
denormFields:
- hour_of_day
- day_of_month
......
......@@ -3,7 +3,7 @@ temporal:
description: Temporal 1
type: date
dateFormat: MM/dd/yyyy HH:mm:ss a
factIndex: -1
factPosition: -1
denormFields:
- hour_of_day
- day_of_month
......
......@@ -3,7 +3,7 @@ temporal:
description: Temporal 1
type: date
dateFormat: MM/dd/yyyy HH:mm:ss a
factIndex: -1
factPosition: -1
denormFields:
- hour_of_day
- day_of_month
......
......@@ -3,7 +3,7 @@ temporal:
description: Temporal 1
type: date
dateFormat: MM/dd/yyyy HH:mm:ss a
factIndex: -1
factPosition: -1
denormFields:
- hour_of_day
- day_of_month
......
......@@ -23,7 +23,7 @@ class ScheduleAndNotifyTest extends SparkTestUtils with BeforeAndAfter {
Set[OutputField](
new IntOutput(name = "intOutput", inputField = new IntegerDimension(name = "intDim", position = 4)),
new DateOutput(name = "dateOutput", inputField = new DateTemporal(name = "dateTemporal",
denormFields = List[String](), algoDef = new ConstantIntDefinition(1000), factIndex = 1, dateFmtStr = dateFmtStr),
denormFields = List[String](), algoDef = new ConstantIntDefinition(1000), factPosition = 1, dateFmtStr = dateFmtStr),
dateFmtStr = dateFmtStr)),
OutputEventFormatters.jsonFormat)
......
......@@ -19,7 +19,7 @@ class DateTemporalTest extends UnitSpec {
assert("test desc" == temporal.description)
assert(temporal.algoDef.asInstanceOf[NaiveBayesDefinition].flatten == 2)
assert(temporal.algoDef.asInstanceOf[NaiveBayesDefinition].polyDegree == 3)
assert(temporal.factIndex == -1)
assert(temporal.factPosition == -1)
assert(.001 == temporal.algoDef.asInstanceOf[NaiveBayesDefinition].lambda)
assert(0 == temporal.denormFields.size)
assert(dateFormat == temporal.dateFormat.toPattern)
......@@ -28,13 +28,13 @@ class DateTemporalTest extends UnitSpec {
test("DateTemporal construction all fields by name in order except description should return proper values") {
val mlDef = new NaiveBayesDefinition(Set(), 2, 3, .001)
val temporal = new DateTemporal(algoDef = mlDef, name = "testTemporal", denormFields = List[String](),
dateFmtStr = dateFormat, factIndex = -1)
dateFmtStr = dateFormat, factPosition = -1)
assert("testTemporal" == temporal.name)
assert("" == temporal.description)
assert(0 == temporal.denormFields.size)
assert(temporal.algoDef.asInstanceOf[NaiveBayesDefinition].flatten == 2)
assert(temporal.algoDef.asInstanceOf[NaiveBayesDefinition].polyDegree == 3)
assert(temporal.factIndex == -1)
assert(temporal.factPosition == -1)
assert(.001 == temporal.algoDef.asInstanceOf[NaiveBayesDefinition].lambda)
assert(dateFormat == temporal.dateFormat.toPattern)
}
......@@ -42,13 +42,13 @@ class DateTemporalTest extends UnitSpec {
test("DateTemporal construction all fields by name out of order should return proper values") {
val mlDef = new LinearRegressionDefinition(Set(), 2, 1, 20, .001)
val temporal = new DateTemporal(algoDef = mlDef, name = "testTemporal", denormFields = List[String](),
dateFmtStr = dateFormat, description = "test desc", factIndex = -1)
dateFmtStr = dateFormat, description = "test desc", factPosition = -1)
assert("testTemporal" == temporal.name)
assert("test desc" == temporal.description)
assert(0 == temporal.denormFields.size)
assert(temporal.algoDef.asInstanceOf[LinearRegressionDefinition].flatten == 2)
assert(1 == temporal.algoDef.asInstanceOf[LinearRegressionDefinition].polyDegree)
assert(temporal.factIndex == -1)
assert(temporal.factPosition == -1)
assert(20 == temporal.algoDef.asInstanceOf[LinearRegressionDefinition].iterations)
assert(.001 == temporal.algoDef.asInstanceOf[LinearRegressionDefinition].stepSize)
assert(dateFormat == temporal.dateFormat.toPattern)
......
......@@ -279,7 +279,7 @@ class InputDefinitionTest extends UnitSpec {
assert(inputDef.temporal.name == "temporal_field")
assert(inputDef.temporal.algoDef.asInstanceOf[LinearRegressionDefinition].flatten == 2)
assert(3 == inputDef.temporal.algoDef.asInstanceOf[LinearRegressionDefinition].polyDegree)
assert(inputDef.temporal.factIndex == -1)
assert(inputDef.temporal.factPosition == -1)
assert(inputDef.temporal.algoDef.isInstanceOf[LinearRegressionDefinition])
assert(20 == inputDef.temporal.algoDef.asInstanceOf[LinearRegressionDefinition].iterations)
assert(Math.abs(.001 - inputDef.temporal.algoDef.asInstanceOf[LinearRegressionDefinition].stepSize) < .00001)
......@@ -335,7 +335,7 @@ class InputDefinitionTest extends UnitSpec {
assert(inputDef.temporal.name == "temporal_field")
assert(inputDef.temporal.algoDef.asInstanceOf[LinearRegressionDefinition].flatten == 2)
assert(3 == inputDef.temporal.algoDef.asInstanceOf[LinearRegressionDefinition].polyDegree)
assert(inputDef.temporal.factIndex == -1)
assert(inputDef.temporal.factPosition == -1)
assert(inputDef.temporal.algoDef.isInstanceOf[LinearRegressionDefinition])
assert(20 == inputDef.temporal.algoDef.asInstanceOf[LinearRegressionDefinition].iterations)
assert(Math.abs(.001 - inputDef.temporal.algoDef.asInstanceOf[LinearRegressionDefinition].stepSize) < .00001)
......@@ -389,7 +389,7 @@ class InputDefinitionTest extends UnitSpec {
assert(inputDef.temporal.name == "temporal_field")
assert(inputDef.temporal.algoDef.asInstanceOf[NaiveBayesDefinition].flatten == 2)
assert(3 == inputDef.temporal.algoDef.asInstanceOf[NaiveBayesDefinition].polyDegree)
assert(inputDef.temporal.factIndex == -1)
assert(inputDef.temporal.factPosition == -1)
assert(inputDef.temporal.algoDef.isInstanceOf[NaiveBayesDefinition])
assert(Math.abs(.001 - inputDef.temporal.algoDef.asInstanceOf[NaiveBayesDefinition].lambda) < .00001)
assert(inputDef.dimensions.size == 1)
......
......@@ -24,7 +24,7 @@ class OutputDefinitionTest extends UnitSpec {
val f2 = new FloatOutput(name = "floatOutput", inputField = new FloatDimension(name = "floatDim", position = 2))
val f3 = new StringOutput(name = "stringOutput", inputField = new StringDimension(name = "stringDim", position = 3))
val f4 = new DateOutput(name = "dateOutput", inputField = new DateTemporal(name = "dateTemporal",
denormFields = List[String](), algoDef = new ConstantIntDefinition(1000), factIndex = 1, dateFmtStr = dateFmtStr),
denormFields = List[String](), algoDef = new ConstantIntDefinition(1000), factPosition = 1, dateFmtStr = dateFmtStr),
dateFmtStr = dateFmtStr)
val fields = Set[OutputField](f1, f2, f3, f4)
......
......@@ -3,7 +3,7 @@ temporal:
description: Date of CM poll
type: date
dateFormat: MM-dd-yyyy-HH:mm:ss
factIndex: -1
factPosition: -1
denormFields:
- day_of_week
- day_of_month
......
......@@ -3,7 +3,7 @@ temporal:
description: Date of CM poll
type: date
dateFormat: MM-dd-yyyy-HH:mm:ss
factIndex: -1
factPosition: -1
denormFields:
- day_of_week
- day_of_month
......
......@@ -3,7 +3,7 @@ temporal:
description: Date of CM poll
type: date
dateFormat: MM-dd-yyyy-HH:mm:ss
factIndex: -1
factPosition: -1
denormFields:
- day_of_week
- day_of_month
......
......@@ -3,7 +3,7 @@ temporal:
description: Date of CM poll
type: date
dateFormat: MM-dd-yyyy-HH:mm:ss
factIndex: -1
factPosition: -1
denormFields:
- day_of_week
- day_of_month
......
......@@ -3,7 +3,7 @@ temporal:
description: Date of CM poll
type: date
dateFormat: MM-dd-yyyy-HH:mm:ss
factIndex: -1
factPosition: -1
denormFields:
- day_of_week
- day_of_month
......
......@@ -3,7 +3,7 @@ temporal:
description: Test temporal
type: date
dateFormat: yyyy-MM-dd HH:mm:ss
factIndex: -1
factPosition: -1
denormFields:
- day_of_week
- day_of_month
......
......@@ -3,7 +3,7 @@ temporal:
description: Time of the customer call
type: date
dateFormat: yyyy-MM-dd HH:mm:ss.SSS
factIndex: -1
factPosition: -1
denormFields:
- day_of_week
- day_of_month
......
......@@ -3,7 +3,7 @@ temporal:
description: Time of the customer call
type: date
dateFormat: yyyy-MM-dd HH:mm:ss.SSS
factIndex: -1
factPosition: -1
denormFields:
- day_of_week
- day_of_month
......
......@@ -3,7 +3,7 @@ temporal:
description: Time of the customer call
type: date
dateFormat: yyyy-MM-dd HH:mm:ss.SSS
factIndex: -1
factPosition: -1
denormFields:
- day_of_week
- day_of_month
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment