/**
* List operations module.
* @module list
*/
import {concat as listAppend, indexOf, slice, includes} from './jsPlatform/list';
import {apply} from './jsPlatform/function';
import {length} from './jsPlatform/object';
import {negateF3, negateF2} from './function/negate';
import {curry, curry2, curry3} from './function/curry';
import {isTruthy, isFalsy} from './boolean';
import {lookup} from './object/lookup';
import {of} from './object/of';
import {isset, isString} from './object/is';
import {typeOf} from './object/typeOf';
import map from './list/map';
import {
sliceFrom, sliceTo, lengths,
toShortest, aggregateArray,
reduceUntil, reduce, reduceRight, lastIndex,
findIndexWhere, findIndexWhereRight, findIndicesWhere,
findWhere, sliceCopy, genericAscOrdering
}
from './list/utils';
export * from './list/range';
export * from './list/utils';
export {map};
export {slice, includes, indexOf, lastIndexOf, push} from './jsPlatform';
export const
/**
* Append two, or more, lists, i.e.,
* @example
* expectEqual(append(take(13, alphabetString), drop(13, alphabetString)), alphabetString); // true
*
* // Another example
* const result = append(
* alphabetStr.split(''),
* alphabetStr.split('')
* ),
* expected = repeat(2, alphabetStr).split('');
*
* shallowEquals(result, expected) === true // `true`
*
* @function module:list.append
* @param [args] {...(Array|String|*)} - One or more lists or list likes (strings etc.).
* @returns {(Array|String|*)} - Same type as list like passed in.
* @curried - Curried at upto 2 arguments.
*/
append = curry2((...args) => apply(listAppend, args)),
/**
* Returns head of list (first item of list).
* @haskellType `head :: [a] -> a`
* @function module:list.head
* @param x {Array|String}
* @returns {*} - First item from list
*/
head = x => x[0],
/**
* Returns last item of list.
* @haskellType `last :: [a] -> a`
* @function module:list.last
* @param xs {Array|String}
* @returns {*}
*/
last = xs => xs[lastIndex(xs)],
/**
* Returns tail part of list (everything after the first item as new list).
* @haskelType `tail :: [a] -> [a]`
* @function module:list.tail
* @param xs {Array|String}
* @returns {Array|String}
*/
tail = xs => sliceFrom(1, xs),
/**
* Returns everything except last item of list as new list.
* @haskellType `init :: [a] -> [a]`
* @function module:list.init
* @param xs {Array|String}
* @returns {Array|String}
*/
init = xs => sliceTo(lastIndex(xs), xs),
/**
* Returns `head` and `tail` of passed in list/string in a tuple.
* @haskellType `uncons :: [a] -> Maybe (a, [a])`
* @function module:list.uncons
* @param xs {Array|String}
* @returns {Array|undefined}
*/
uncons = xs => !xs || length(xs) === 0 ? undefined : [head(xs), tail(xs)],
/**
* Returns `tail` and `head` of passed in list/string in a tuple.
* @haskellType `unconsr :: [a] -> Maybe ([a], a)`
* @function module:list.unconsr
* @param xs {Array|String}
* @returns {Array|String|*|undefined}
*/
unconsr = xs => !xs || length(xs) === 0 ? undefined : [init(xs), last(xs)],
/**
* Concatenates all the elements of a container of lists.
* @haskellType `concat :: Foldable t => t [a] -> [a]`
* @function module:list.concat
* @param xs {Array}
* @returns {Array}
*/
concat = xs => {
switch (length(xs)) {
case undefined:
case 0:
return [];
case 1:
const item0 = xs[0];
return item0 && item0.slice ? sliceCopy(item0) : item0;
case 2:
default:
return apply(append, xs);
}
},
/**
* Map a function over all the elements of a container and concatenate the resulting lists.
* @haskellType `concatMap :: Foldable t => (a -> [b]) -> t a -> [b]`
* @function module:list.concatMap
* @param fn {Function}
* @param foldableOfA {Array}
* @returns {Array}
*/
concatMap = curry((fn, foldableOfA) => concat(map(fn, foldableOfA))),
/**
* Returns a copy of the passed in list reverses.
* @haskellType `reverse :: [a] -> [a]`
* @function module:list.reverse
* @param xs {Array|String}
* @returns {Array|String}
*/
reverse = xs => {
if (!isset(xs) || !xs.length) {
return xs;
}
let out = of(xs),
i = xs.length - 1;
switch (typeOf(xs)) {
case 'String':
for (; i >= 0; i -= 1) {
out += xs[i];
}
return out;
default:
for (; i >= 0; i -= 1) {
out.push(xs[i]);
}
return out;
}
},
/**
* Takes an element and a list and `intersperses' that element between the
* elements of the list.
* @function module:list.intersperse
* @note In our version of the function javascript is loosely typed so,
* so is our function (to much overhead to make it typed) so `between` can be any value.
* @param between {*} - Should be of the same type of elements contained in list.
* @param arr {Array|String} - List.
* @returns {Array|String}
*/
intersperse = curry((between, xs) => {
if (!xs || !xs.length) {
return xs;
}
const limit = xs.length,
lastInd = limit - 1;
let out = of(xs),
i = 0;
if (isString(xs)) {
for (; i < limit; i += 1) {
out += i === lastInd ?
xs[i] : xs[i] + between;
}
return out;
}
for (; i < limit; i += 1) {
if (i === lastInd) {
out.push(xs[i]);
} else {
out.push(xs[i], between);
}
}
return out;
}),
/**
* `intercalate xs xss` is equivalent to (concat (intersperse xs xss)). It inserts the list xs in between the lists in xss and concatenates the result.
* @haskellType `intercalate :: [a] -> [[a]] -> [a]`
* @function module:list.intercalate
* @param xs {Array|String}
* @param xss {Array|String}
* @returns {Array|String}
*/
intercalate = curry((xs, xss) => {
if (isString(xss)) {
return intersperse(xs, xss);
}
return concat(intersperse(xs, xss));
}),
/**
* Transposes rows and columns into lists by index; E.g.,
* Haskell example:
* ```
* transpose [[1,2,3],[4,5,6]] == [[1,4],[2,5],[3,6]]
*
* -- Notice the shorter arrays are ignored after their last index is copied over:
* transpose [[10,11],[20],[],[30,31,32]] == [[10,20,30],[11,31],[32]]
* ```
* @note from columns to rows.
* @note Empty lists are ignored.
* @haskellType `transpose :: [[a]] -> [[a]]`
* @function module:list.transpose
* @param xss {Array}
* @returns {Array}
*/
transpose = xss => {
let numLists = length(xss),
ind = 0, ind2;
if (!numLists) {
return [];
}
const listLengths = apply(lengths, xss),
longestListLen = maximum(listLengths),
outLists = [];
for (; ind < longestListLen; ind += 1) {
const outList = [];
for (ind2 = 0; ind2 < numLists; ind2 += 1) {
if (listLengths[ind2] < ind + 1) {
continue;
}
outList.push(xss[ind2][ind]);
}
outLists.push(outList);
}
return filter(x => length(x) > 0, outLists);
},
/**
* Generates 2^n sub-sequences for passed in sequence (string/list) (`n` is
* the length of the passed in sequence so: 2^length(xs)).
* Note: The return value doubles per index/character passed in so use with caution!
* Also note that for 2^16 (or for a sequence of 16 characters) this algorithm
* will generate 65536 sub-sequences! So caution should be taken to not
* use this with sequences above a certain length on certain platform (the browser thread in specific).
* @function module:list.subsequences
* @jsperftest https://jsperf.com/subsequences
* @param xs {Array|String}
* @returns {Array.<Array>}
*/
subsequences = xs => {
const listLen = length(xs),
len = Math.pow(2, listLen),
out = [];
for (let i = 0; i < len; i += 1) {
let entry = [];
for (let j = 0; j < listLen; j += 1) {
if (i & (1 << j)) {
entry.push(xs[j]);
}
}
out.push(entry);
}
return out;
},
/**
* Returns an array with the given indices swapped.
* @function module:list.swapped
* @param ind1 {Number}
* @param ind2 {Number}
* @param list {Array}
* @returns {Array} - Copy of incoming with swapped values at indices.
*/
swapped = curry((ind1, ind2, list) => {
const out = sliceCopy(list),
tmp = out[ind1];
out[ind1] = out[ind2];
out[ind2] = tmp;
return out;
}),
/**
* Returns a list of permutations for passed in list.
* Use caution with lists above a length of 15 (will take long due to nature of
* algorithm).
* @function module:list.permutations
* @param xs {Array} - List.
* @returns {Array<Array|String|*>} - Array of permutations.
*/
permutations = xs => {
const limit = length(xs);
if (!limit || limit === 1) {
return [xs];
}
let list = sliceCopy(xs),
c = repeat(limit, 0),
i = 0;
const out = [list];
for (; i < limit; i++) {
if (c[i] < i) {
list = swapped(i % 2 === 0 ? 0 : c[i], i, list);
out.push(list);
c[i] += 1;
i = 0;
continue;
}
c[i] = 0;
}
return out;
},
/**
* Left associative fold. Reduces a container of elements down by the given operation (same as [].reduce).
* @function module:list.foldl
* @param fn {Function}
* @param zero {*} - Aggregator.
* @param functor {Array}
* @returns {*} - Whatever type is lastly returned from `fn`.
*/
foldl = reduce,
/**
* Right associative fold. Reduces a container of elements down by the given operation (same as [].reduceRight).
* @function module:list.foldr
* @param fn {Function}
* @param zero {*} - Aggregator.
* @param functor {Array}
* @returns {*} - Whatever type is lastly returned from `fn`.
*/
foldr = reduceRight,
/**
* A variant of `foldl` except that this one doesn't require the starting point. The starting point/value will be pulled
* out from a copy of the container.
* @function module:list.foldl1
* @param op {Function}
* @param xs {Array}
* @returns {*} - Whatever type is lastly returned from `op`.
*/
foldl1 = curry((op, xs) => {
const parts = uncons(xs);
return !parts ? [] : reduce(op, parts[0], parts[1]);
}),
/**
* A variant of `foldr` except that this one doesn't require the starting point/value. The starting point/value will be pulled
* out from a copy of the container.
* @function module:list.foldr1
* @param op {Function}
* @param xs {Array}
* @returns {*} - Whatever type is lastly returned from `op`.
*/
foldr1 = curry((op, xs) => {
const parts = unconsr(xs);
return !parts ? [] : reduceRight(op, parts[1], parts[0]);
}),
/**
* Performs a map then a reduce all in one (from left-to-right). Returns a tuple
* containing the aggregated value and the result of mapping the passed in function on passed in list.
* @function module:list.mapAccumL
* @param op {Function} - Function<aggregator, item, index> : [aggregated, mapResult]
* @param zero {*} - An instance of the passed in list type used to aggregateArray on.
* @param xs {Array} - list type.
* @return {Array} - [aggregated, list]
*/
mapAccumL = curry((op, zero, xs) => {
const list = sliceCopy(xs),
limit = length(xs);
if (!limit) {
return [zero, list];
}
let ind = 0,
agg = zero,
mapped = [],
tuple;
for (; ind < limit; ind++) {
tuple = op(agg, list[ind], ind);
agg = tuple[0];
mapped = tuple[1];
}
return [agg, mapped];
}),
/**
* Performs a map and a reduce all in one (from right-to-left). Returns a tuple
* containing the aggregated value and the result of mapping the passed in function on passed in list.
* @function module:list.mapAccumR
* @param op {Function} - Function<aggregator, item, index> : [aggregated, mapResult]
* @param zero {*} - An instance of the passed in list type used to aggregateArray on.
* @param xs {Array} - list type.
* @return {Array} - [aggregated, list]
*/
mapAccumR = curry((op, zero, xs) => {
const list = sliceCopy(xs),
limit = length(xs);
if (!limit) {
return [zero, list];
}
let ind = limit - 1,
agg = zero,
mapped = [],
tuple;
for (; ind >= 0; ind--) {
tuple = op(agg, list[ind], ind);
agg = tuple[0];
mapped = tuple[1];
}
return [agg, mapped];
}),
/**
* iterate f x returns an infinite list of repeated applications of f to x.
* @function module:list.iterate
* @example `iterate(5, f, x) == [x, f(x), f(f(x)), ...]`
* @param limit {Number}
* @param op {Function} - Operation.
* @param x {*} - Starting point.
* @returns {*}
*/
iterate = curry((limit, op, x) => {
let ind = 0,
out = [],
lastX = x;
for (; ind < limit; ind += 1) {
out.push(lastX);
lastX = op(lastX, ind);
}
return out;
}),
/**
* Repeats `x` `limit` number of times.
* @function module:list.repeat
* @param limit {Number}
* @param x {*}
* @return {Array}
*/
repeat = curry((limit, x) => iterate(limit, a => a, x)),
/**
* Same as `repeat` due to the nature of javascript (see haskell version for usage).
* @function module:list.replicate
* @param limit {Number}
* @param x {*}
* @return {Array}
*/
replicate = repeat,
/**
* Replicates a list `limit` number of times and appends the results (concat)
* @function module:list.cycle
* @param limit {Number}
* @param xs {Array}
* @returns {Array}
*/
cycle = curry((limit, xs) => concat(replicate(limit, xs))),
/**
* Unfolds a value into a list of somethings.
* @haskellType `unfoldr :: (b -> Maybe (a, b)) -> b -> [a]`
* @function module:list.unfoldr
* @param op {Function} - Operation to perform (should return a two component tuple (item to aggregateArray and item to unfold in next iteration).
* @param x {*} - Starting parameter to unfold from.
* @returns {Array} - An array of whatever you return from `op` yielded.
*/
unfoldr = curry((op, x) => {
let ind = 0,
out = [],
resultTuple = op(x, ind, out);
while (resultTuple) {
out.push(resultTuple[0]);
resultTuple = op(resultTuple[1], ++ind, out);
}
return out;
}),
/**
* Finds index in string or list (alias for `findIndex`).
* @function module:list.findIndex
* @param pred {Function} - Predicate<element, index, arr>.
* @param arr {Array|String}
* @returns {Number} - `-1` if predicate not matched else `index` found
*/
findIndex = findIndexWhere,
/**
* @function module:list.findIndices
* @param pred {Function}
* @param xs {Array} - list or list like.
* @returns {Array|undefined}
*/
findIndices = findIndicesWhere,
/**
* @function module:list.elemIndex
* @param x {*} - Element to search for.
* @param xs {Array} - list or list like.
* @returns {*}
*/
elemIndex = curry((x, xs) => {
const foundInd = indexOf(x, xs);
return foundInd !== -1 ? foundInd : undefined;
}),
/**
* @function module:list.elemIndices
* @param value {*} - Element to search for.
* @param xs {Array} - list or list like.
* @returns {*}
*/
elemIndices = curry((value, xs) => findIndices(x => x === value, xs)),
/**
* Takes `n` items from start of list to `limit` (exclusive).
* @function module:list.take
* @param list {Array|String}
* @param limit {Number}
* @returns {String|Array} - Passed in type's type
*/
take = sliceTo,
/**
* Drops `n` items from start of list to `count` (exclusive).
* @function module:list.drop
* @param list {Array|String}
* @param count {Number}
* @returns {String|Array} - Passed in type's type
*/
drop = sliceFrom,
/**
* Splits `x` in two at given `index` (exclusive (includes element/character at
* given index in second part of returned list)).
* @function module:list.splitAt
* @param ind {Number} - Index to split at.
* @param list {Array|String} - functor (list or string) to split.
* @returns {Array|String} - List like type passed
*/
splitAt = (ind, list) => [sliceTo(ind, list), sliceFrom(ind, list)],
/**
* Gives an list with passed elements while predicate was true.
* @function module:list.takeWhile
* @param pred {Function} - Predicate<*, index, list|string>
* @param list {Array|String}
* @returns {Array}
*/
takeWhile = curry((pred, list) =>
reduceUntil(
negateF3(pred), // predicate
isString(list) ?
(agg, x) => agg + x :
aggregateArray, // operation
of(list), // aggregate
list
)),
/**
* Returns an list without elements that match predicate.
* @function module:list.dropWhile
* @param pred {Function} - Predicate<*, index, list|string>
* @param list {Array|String}
* @refactor
* @returns {Array|String}
*/
dropWhile = curry((pred, list) => {
const limit = length(list),
splitPoint =
findIndexWhere(
(x, i, xs) => !pred(x, i, xs),
list
);
return splitPoint === -1 ?
sliceFrom(limit, list) :
slice(splitPoint, limit, list);
}),
/**
* @function module:list.dropWhileEnd
* @param pred {Function} - Predicate<*, index, list|string>
* @param list {Array|String}
* @refactor
* @returns {Array|String}
*/
dropWhileEnd = curry((pred, list) => {
const splitPoint =
findIndexWhereRight(
(x, i, xs) => !pred(x, i, xs),
list
);
if (splitPoint === -1) {
return of(list);
}
return sliceTo(splitPoint + 1, list);
}),
/**
* Gives you the `span` of items matching predicate
* and items not matching predicate; E.g., Gives an
* array of arrays; E.g., [[matching-items], [non-matching-items]]
* @function list.span
* @param pred {Function} - List predicate (`(x, i, list) => bool`)
* @param list {Array|String}
* @returns {(Array<Array<*>>|Array<String>)}
* @type {Function}
*/
span = curry((pred, list) => {
const splitPoint = findIndexWhere(negateF3(pred), list);
return splitPoint === -1 ?
[sliceFrom(0, list), of(list)] :
splitAt(splitPoint, list);
}),
/**
* breakOnList, applied to a predicate p and a list xs, returns a tuple
* where first element is longest prefix (possibly empty) of xs of elements
* that do not satisfy p and second element is the remainder of the list:
* @haskellExample
* Replace `break` with `breakOnList` for our version.
* ```
* breakOnList (> 3) [1,2,3,4,1,2,3,4] == ([1,2,3],[4,1,2,3,4])
* breakOnList (< 9) [1,2,3] == ([],[1,2,3])
* breakOnList (> 9) [1,2,3] == ([1,2,3],[])
* ```
* @function module:list.breakOnList
* @param pred {Function}
* @param list {Array|String|*}
* @returns {Array}
*/
breakOnList = curry((pred, list) => {
const splitPoint = findIndexWhere(negateF3(pred), list);
return splitPoint === -1 ?
[of(list), sliceFrom(0, list)] : reverse(splitAt(splitPoint, list));
}),
/**
* Gets item at index.
* @function module:list.at
* @param ind {Number} - Index.
* @param xs {Array} - list or list like.
* @returns {*|undefined} - Item or `undefined`.
*/
at = lookup,
/**
* Find an item in structure of elements based on given predicate (`pred`).
* @function module:list.find
* @param pred {Function}
* @param xs {Array} - list or list like.
* @returns {*} - Found item.
*/
find = findWhere,
/**
* For each function (same as `[].forEach` except in functional format).
* @function module:list.forEach
* @param fn {Function} - Operation (`(element, index, list) => {...}`, etc.)
* @param xs {(Array|String)}
* @returns {void}
*/
forEach = curry((fn, list) => {
const limit = length(list);
if (!limit) {
return;
}
let ind = 0;
for (; ind < limit; ind += 1) {
fn(list[ind], ind, list);
}
}),
/**
* Filters a structure of elements using given predicate (`pred`) (same as `[].filter`).
* @function module:list.filter
* @param pred {Function}
* @param xs {Array} - list or list like.
* @returns {Array} - Structure of filtered elements.
*/
filter = curry((pred, xs) => {
let ind = 0,
limit = length(xs),
out = [];
if (!limit) {
return out;
}
for (; ind < limit; ind++) {
if (pred(xs[ind], ind, xs)) {
out.push(xs[ind]);
}
}
return out;
}),
/**
* Partitions a list on a predicate; Items that match predicate are in first list in tuple; Items that
* do not match the tuple are in second list in the returned tuple.
* Essentially `[filter(p, xs), filter(negateF3(p), xs)]`.
* @function module:list.partition
* @param pred {Function} - Predicate<item, index, originalArrayOrString>
* @param list {Array}
* @returns {Array|String} - Tuple of arrays or strings (depends on incoming list (of type list or string)).
*/
partition = curry((pred, list) =>
!length(list) ?
[[], []] :
[filter(pred, list), filter(negateF3(pred), list)]),
/**
* Returns a boolean indicating whether an element exists in given structure of elements.
* @function module:list.elem
* @param element {*}
* @param xs {Array}
* @returns {Boolean}
*/
elem = includes,
/**
* The opposite of `elem` - Returns a boolean indicating whether an element exists in given list.
* @function module:list.notElem
* @param element {*}
* @param xs {Array}
* @returns {Boolean}
*/
notElem = negateF2(includes),
/**
* Checks if list `xs1` is a prefix of list `xs2`
* @function module:list.isPrefixOf
* @param xs1 {Array|String|*}
* @param xs2 {Array|String|*}
* @returns {boolean}
*/
isPrefixOf = curry((xs1, xs2) => {
const limit1 = length(xs1),
limit2 = length(xs2);
if (limit2 < limit1 || !limit1 || !limit2 || indexOf(xs1[0], xs2) === -1) {
return false;
}
let ind = 0;
for (; ind < limit1; ind++) {
if (xs1[ind] !== xs2[ind]) {
return false;
}
}
return true;
}),
/**
* Checks if list `xs1` is a suffix of list `xs2`
* @function module:list.isSuffixOf
* @param xs1 {Array|String|*}
* @param xs2 {Array|String|*}
* @returns {boolean}
*/
isSuffixOf = curry((xs1, xs2) => {
const limit1 = length(xs1),
limit2 = length(xs2);
if (limit2 < limit1 || !limit1 || !limit2 || indexOf(xs1[0], xs2) === -1) {
return false;
}
let ind1 = limit1 - 1,
ind2 = limit2 - 1;
for (; ind1 >= 0; ind1--) {
if (xs1[ind1] !== xs2[ind2]) {
return false;
}
ind2 -= 1;
}
return true;
}),
/**
* Checks if list `xs1` is an infix of list `xs2`
* @function module:list.isInfixOf
* @param xs1 {Array|String|*}
* @param xs2 {Array|String|*}
* @returns {boolean}
*/
isInfixOf = curry((xs1, xs2) => {
const limit1 = length(xs1),
limit2 = length(xs2);
if (limit2 < limit1 || !limit1 || !limit2) {
return false;
}
let ind1,
foundLen,
ind = 0;
for (; ind < limit2; ind += 1) {
foundLen = 0;
for (ind1 = 0; ind1 < limit1; ind1 += 1) {
if (xs2[ind1 + ind] === xs1[ind1]) {
foundLen += 1;
}
if (foundLen === limit1) {
return true;
}
}
}
return false;
}),
/**
* Checks if list `xs1` is a sub-sequence of list `xs2`
* @function module:list.isSubsequenceOf
* @param xs1 {Array|String|*}
* @param xs2 {Array|String|*}
* @returns {boolean}
*/
isSubsequenceOf = curry((xs1, xs2) => {
const len = Math.pow(2, length(xs2)),
lenXs1 = length(xs1);
let foundLen,
i;
for (i = 0; i < len; i += 1) {
foundLen = 0;
for (let j = 0; j < len; j += 1) {
if (i & (1 << j) && indexOf(xs2[j], xs1) > -1) {
foundLen += 1;
}
if (foundLen === lenXs1) {
return true;
}
}
}
return false;
}),
/**
* The group function takes a list and returns a list of lists such that
* the concatenation of the result is equal to the argument. Moreover, each
* sublist in the result contains only equal elements. For example,
* `group "Mississippi" = ["M","i","ss","i","ss","i","pp","i"]`
* It is a special case of groupBy, which allows the programmer to supply
* their own equality test.
* @haskellType `group :: Eq a => [a] -> [[a]]`
* @function module:list.group
* @param xs {Array|String}
* @returns {Array<Array|String|*>|*}
*/
group = xs => groupBy((a, b) => a === b, xs),
/**
* Allows you to group items in a list based on your supplied equality check.
* @note Sames `group` but allows you to specify equality operation.
* @haskellType `groupBy :: (a -> a -> Bool) -> [a] -> [[a]]`
* @function module:list.groupBy
* @param equalityOp {Function}
* @param xs {Array}
* @returns {*}
*/
groupBy = curry((equalityOp, xs) => {
const limit = length(xs);
if (!limit) {
return sliceCopy(xs);
}
let ind = 0,
prevItem,
item,
predOp = x => {
if (equalityOp(x, prevItem)) {
ind++;
}
if (equalityOp(x, item)) {
prevItem = x;
return true;
}
return false;
},
agg = [];
for (; ind < limit; ind += 1) {
item = xs[ind];
agg.push(takeWhile(predOp, slice(ind, limit, xs)));
}
return agg;
}),
/**
* The inits function returns all initial segments of the argument, shortest first. For example,
* ```
* shallowEquals(inits('abc'), ['','a','ab','abc'])
* ```
* @function module:list.inits
* @haskellType `inits :: [a] -> [[a]]`
* @param xs {Array}
* @returns {Array}
*/
inits = xs => {
let limit = length(xs),
ind = 0,
agg = [];
if (!limit) {
return [];
}
for (; ind <= limit; ind += 1) {
agg.push(sliceTo(ind, xs));
}
return agg;
}, //map(list => init(list), xs),
/**
* The inits function returns all initial segments of the argument, shortest first. For example,
* ```
* shallowEquals(tails('abc'), ['abc', 'bc', 'c',''])
* ```
* @function module:list.tails
* @haskellType `tails :: [a] -> [[a]]`
* @param xs {Array}
* @returns {Array}
*/
tails = xs => {
let limit = length(xs),
ind = 0,
agg = [];
if (!limit) {
return [];
}
for (; ind <= limit; ind += 1) {
agg.push(slice(ind, limit, xs));
}
return agg;
}, //map(list => tail(list), xs),
/**
* Strips prefix list from given list
* @function module:list.stripPrefix
* @param prefix {Array|String|*}
* @param list {Array|string|*}
* @returns {Array|*}
*/
stripPrefix = curry((prefix, list) =>
isPrefixOf(prefix, list) ?
splitAt(length(prefix), list)[1] :
sliceCopy(list)),
/**
* zip takes two lists and returns a list of corresponding pairs.
* If one input list is short, excess elements of the longer list are discarded.
* @haskellType `zip :: [a] -> [b] -> [(a, b)]`
* @function module:list.zip
* @param arr1 {Array}
* @param arr2 {Array}
* @returns {Array<Array<*,*>>}
*/
zip = curry((arr1, arr2) => {
if (!length(arr1) || !length(arr2)) {
return [];
}
const [a1, a2] = toShortest(arr1, arr2);
return reduce((agg, item, ind) =>
aggregateArray(agg, [item, a2[ind]]),
[], a1);
}),
/**
* zipN takes one or more lists and returns a list containing lists of all indices
* at a given index, index by index.
* If one input list is short, excess elements of the longer list are discarded.
* @function module:list.zipN
* @param lists {Array|String} - One ore more lists of the same type.
* @returns {Array}
*/
zipN = curry2((...lists) => {
const trimmedLists = apply(toShortest, lists);
return reduce((agg, item, ind) =>
aggregateArray(agg, map(xs => xs[ind], trimmedLists)),
[], trimmedLists[0]);
}),
/**
* @haskellType `zip3 :: [a] -> [b] -> [c] -> [(a, b, c)]`
* @function module:list.zip3
* @param arr1 {Array}
* @param arr2 {Array}
* @param arr3 {Array}
* @returns {Array<Array<*,*>>}
*/
zip3 = curry((arr1, arr2, arr3) => zipN(arr1, arr2, arr3)),
/**
* @haskellType `zip4 :: [a] -> [b] -> [c] -> [d] -> [(a, b, c, d)]`
* @function module:list.zip4
* @param arr1 {Array}
* @param arr2 {Array}
* @param arr3 {Array}
* @param arr4 {Array}
* @returns {Array<Array<*,*>>}
*/
zip4 = curry((arr1, arr2, arr3, arr4) => zipN(arr1, arr2, arr3, arr4)),
/**
* @haskellType `zip5 :: [a] -> [b] -> [c] -> [d] -> [e] -> [(a, b, c, d, e)]`
* @function module:list.zip5
* @param arr1 {Array}
* @param arr2 {Array}
* @param arr3 {Array}
* @param arr4 {Array}
* @param arr5 {Array}
* @returns {Array<Array<*,*>>}
*/
zip5 = curry((arr1, arr2, arr3, arr4, arr5) => zipN(arr1, arr2, arr3, arr4, arr5)),
/**
* zipWith :: (a -> b -> c) -> [a] -> [b] -> [c]
* zipWith generalises zip by zipping with the function given as the
* first argument, instead of a function tupling function (function that returns a tuple). For example,
* zipWith (+) is applied to two lists to produce the list of corresponding sums.
* @note `_|_` means bottom or perpetual (@see
* - https://wiki.haskell.org/Bottom
* - https://stackoverflow.com/questions/19794681/what-does-this-syntax-mean-in-haskell-or
* )
* @example
* ```
* zipWith f [] _|_ = []
* ```
* @haskellType `zipWith :: (a -> b -> c) -> [a] -> [b] -> [c]`
* @function module:list.zipWith
* @param op {Function} - Takes two parts of a tuple and returns a tuple.
* E.g., ` op :: a -> b -> (a, b)`
* @param xs1 {Array}
* @param xs2 {Array}
* @returns {Array<Array<*,*>>}
*/
zipWith = curry((op, xs1, xs2) => {
if (!length(xs1) || !length(xs2)) {
return [];
}
const [a1, a2] = toShortest(xs1, xs2);
return reduce((agg, item, ind) =>
aggregateArray(agg, op(item, a2[ind])),
[], a1);
}),
/**
* Zips all given lists with tupling function. Note: Haskell types do not have
* a way (that I know of) to show one or more for params in a function so `@haskellType` below
* is left there for general purpose not for exactness as is told by aforementioned.
* @haskellType `zipWithN :: (a -> b -> c) -> [a] -> [b] -> [c]` - Where `N` is the number
* of lists to zip.
* @function module:list.zipWithN
* @param op {Function} - Takes expected number of parts for tuple and returns a tuple
* of said parts:
* E.g., ` op :: a -> b -> c -> (a, b, c)`
* @param lists ...{Array}
* @returns {Array<Array<*,*>>}
*/
zipWithN = curry3((op, ...lists) => {
const trimmedLists = apply(toShortest, lists),
lenOfTrimmed = length(trimmedLists);
if (!lenOfTrimmed) {
return [];
}
else if (lenOfTrimmed === 1) {
return sliceTo(length(trimmedLists[0]), trimmedLists[0]);
}
return reduce((agg, item, ind) =>
aggregateArray(agg, apply(op, map(xs => xs[ind], trimmedLists))),
[], trimmedLists[0]);
}),
/**
* Zips 3 lists with tupling function.
* @haskellType `zipWith3 :: (a -> b -> c -> d) -> [a] -> [b] -> [c] -> [d]`
* @function module:list.zipWith3
* @param op {Function} - Takes expected number of parts for tuple and returns a tuple
* of said parts:
* E.g., ` op :: a -> b -> c -> (a, b, c)`
* @param xs1 {Array}
* @param xs2 {Array}
* @param xs3 {Array}
* @returns {Array<Array<*,*>>}
*/
zipWith3 = curry((op, xs1, xs2, xs3) => zipWithN(op, xs1, xs2, xs3)),
/**
* Zips 4 lists with tupling function.
* @haskellType `zipWith4 :: (a -> b -> c -> d -> e) -> [a] -> [b] -> [c] -> [d] -> [e]`
* @function module:list.zipWith4
* @param op {Function} - Takes expected number of parts for tuple and returns a tuple
* of said parts:
* E.g., ` op :: a -> b -> c -> d -> (a, b, c, d)`
* @param xs1 {Array}
* @param xs2 {Array}
* @param xs3 {Array}
* @param xs4 {Array}
* @returns {Array<Array<*,*>>}
*/
zipWith4 = curry((op, xs1, xs2, xs3, xs4) => zipWithN(op, xs1, xs2, xs3, xs4)),
/**
* Zips 5 lists.
* @haskellType `zipWith5 :: (a -> b -> c -> d -> e -> f) -> [a] -> [b] -> [c] -> [d] -> [e] -> [f]`
* @function module:list.zipWith5
* @param op {Function} - Takes expected number of parts for tuple and returns a tuple
* of said parts:
* E.g., ` op :: a -> b -> c -> d -> e -> (a, b, c, d, e)`
* @param xs1 {Array}
* @param xs2 {Array}
* @param xs3 {Array}
* @param xs4 {Array}
* @param xs5 {Array}
* @returns {Array<Array<*,*>>}
*/
zipWith5 = curry((op, xs1, xs2, xs3, xs4, xs5) => zipWithN(op, xs1, xs2, xs3, xs4, xs5)),
/**
* unzip transforms a list of pairs into a list of first components and a list of second components.
* @haskellType `unzip :: [(a, b)] -> ([a], [b])`
* @function module:list.unzip
* @param arr {Array|*}
* @returns {Array|*}
*/
unzip = foldl((agg, item) => {
agg[0].push(item[0]);
agg[1].push(item[1]);
return agg;
}, [[], []]),
/**
* unzip transforms a list of pairs into a list of first components and a list of second components.
* @sudoHaskellType `unzipN :: [(a, b, ...x)] -> ([a], [b], ...[x])`
* @function module:list.unzipN
* @param list {Array|*} - List of tuples (lists).
* @returns {Array|*}
*/
unzipN = list => {
if (!length(list)) {
return [];
}
const lenItem0 = length(list[0]);
let zero = lenItem0 ?
unfoldr(numLists => numLists-- ? [[], numLists] : undefined, lenItem0) :
[];
return foldl((agg, item) => {
agg.forEach((outList, ind) => outList.push(item[ind]));
return agg;
}, zero, list);
},
/**
* Returns true if any item in container passes predicate `p`.
* @function module:list.any
* @param p {Function} - Predicate.
* @param xs {Array|String}
* @returns {Boolean}
*/
any = curry((p, xs) => {
let ind = 0,
limit = length(xs);
if (!limit) {
return false;
}
for (; ind < limit; ind += 1) {
if (p(xs[ind])) {
return true;
}
}
return false;
}),
/**
* Returns true if all items in container pass predicate `p`.
* @function module:list.all
* @param p {Function} - Predicate.
* @param xs {Array|String}
* @returns {Boolean}
*/
all = curry((p, xs) => {
const limit = length(xs);
let ind = 0;
if (!limit) {
return false;
}
for (; ind < limit; ind++) {
if (!p(xs[ind], ind, xs)) {
return false;
}
}
return true;
}),
/**
* Conjuction of container of bools (or truthy and/or falsy values); Returns
* `true` if all in container are 'truthy' else returns `false`
* @function module:list.and
* @param xs {Array|String}
* @returns {Boolean}
*/
and = xs => all(isTruthy, xs),
/**
* Returns a boolean indicating whether any item in container is 'truthy' or not.
* **Note** The haskell type for this function only takes two items, but here
* we allow the passing of more than one item (may change later to adhere to the haskell type).
* @function module:list.or
* @haskellType `or :: Bool -> Bool -> Bool`
* @param xs {Array|String}
* @returns {Boolean}
*/
or = xs => any(isTruthy, xs),
/**
* Returns a boolean indicating whether all items in container are 'falsy' or not.
* **Note** The haskell type for this function only takes two items, but here
* we allow the passing of more than one item (may change later to adhere to the haskell type).
* @function module:list.not
* @haskellType `not :: Bool -> Bool`
* @param xs {Array|String}
* @returns {Boolean}
*/
not = xs => all(isFalsy, xs),
/**
* Computes the sum of the numbers of a structure.
* @function module:list.sum
* @haskellType `sum :: (List t, Num a) => t a -> a`
* @param list {Array|String}
* @returns {Number}
*/
sum = list => foldl((agg, x) => agg + x, 0, list),
/**
* Computes the product of the numbers of a structure.
* @function module:list.product
* @haskellType `product :: (List t, Num a) => t a -> a`
* @param list {Array|String}
* @returns {Number}
*/
product = list => foldl((agg, x) => agg * x, 1, list),
/**
* Returns the largest element in a non-empty structure of elements.
* @function module:list.maximum
* @haskellType `maximum :: forall a . Ord a => t a -> a`
* @param list {Array|String}
* @returns {*} - Whatever type the array is made of (if any).
*/
maximum = list => last(sortBy(genericAscOrdering, list)),
/**
* Returns the smallest element in a non-empty structure of elements.
* @function module:list.minimum
* @haskellType `minimum :: forall a . Ord a => t a -> a`
* @param list {Array|String}
* @returns {*} - Whatever type the array is made of (if any).
*/
minimum = list => head(sortBy(genericAscOrdering, list)),
/**
* scanl is similar to foldl, but returns a list of successive reduced values from the left:
* ```
* scanl f z [x1, x2, ...] == [z, z `f` x1, (z `f` x1) `f` x2, ...]
* ```
* Also note that:
* ```
* last (scanl f z xs) == foldl f z xs.
* ```
* @function module:list.scanl
* @param fn {Function}
* @param zero {*}
* @param xs {Array}
* @returns {Array|*}
*/
scanl = curry((fn, zero, xs) => {
if (!xs || !length(xs)) {
return [];
}
const limit = length(xs);
let ind = 0,
result = zero,
out = [];
while (ind < limit) {
result = fn(result, xs[ind], ind, xs);
out.push(result);
ind++;
}
return out;
}),
/**
* `scanl1` is a variant of `scanl` that has no starting value argument:
* `shallowCompare(scanl1(fn, [x1, x2, ...]), [x1, fn(x1, x2), ...]) // true`
* @function module:list.scanl1
* @param fn {Function}
* @param xs {Array}
* @returns {Array|*}
*/
scanl1 = curry((fn, xs) => {
if (!xs || !xs.length) {
return [];
}
return scanl(fn, head(xs), tail(xs));
}),
/**
* Same as `scanl` but from the right (similiar to `foldr`'s relationship to 'foldl').
* Note also `scanr`'s relationship ot `foldr`:
* `head (scanr(fn, z, xs)) === foldr(fn, z, xs).
* @function module:list.scanr
* @param fn {Function}
* @param zero {*}
* @param xs {Array}
* @returns {Array|*}
*/
scanr = curry((fn, zero, xs) => {
if (!xs || !length(xs)) {
return [];
}
const limit = length(xs);
let ind = limit - 1,
result = xs[0],
out = [];
while (ind > -1) {
result = fn(result, xs[ind], ind, xs);
out.push(result);
ind--;
}
return out;
}),
/**
* Same as `scanr` but takes no zero/accumulator value.
* @function module:list.scanr1
* @param fn {Function}
* @param xs {Array}
* @returns {Array|*}
*/
scanr1 = curry((fn, xs) => {
if (!xs || !xs.length) {
return [];
}
return scanr(fn, last(xs), init(xs));
}),
/**
* The nub function removes duplicate elements from a list.
* In particular, it keeps only the first occurrence of each element.
* (The name nub means `essence'.) It is a special case of nubBy, which
* allows the programmer to supply their own equality test.
* ```shallowCompare( nub ([1,2,3,4,3,2,1,2,4,3,5]), [1,2,3,4,5] )```
* @function module:list.nub
* @param list {Array|String|*}
* @returns {Array}
*/
nub = list => nubBy((a, b) => a === b, list),
/**
* `remove(x, xs)` removes the first occurrence of `x` from `xs`.
* For example, `remove('a', 'banana') === 'bnana';`
* @function module:list.remove
* @param x {*}
* @param list {Array|String|*}
* @returns {Array}
*/
remove = curry((x, list) => removeBy((a, b) => a === b, x, list)),
/**
* The sort function implements a stable sorting algorithm.
* It is a special case of sortBy, which allows the programmer
* to supply their own comparison function.
* ```shallowCompare(sort ([1,6,4,3,2,5]), [1,2,3,4,5,6]) // true```
* @function module:list.sort
* @param xs {Array|String|*}
* @returns {Array}
*/
sort = xs => sortBy(genericAscOrdering, xs),
/**
* Sort a list by comparing the results of a key function applied to each
* element. sortOn f is equivalent to sortBy (comparing f), but has the
* performance advantage of only evaluating f once for each element in the
* input list. This is called the decorate-sort-undecorate paradigm, or
* Schwartzian transform.
*
* Elements are arranged from from lowest to highest, keeping duplicates
* in the order they appeared in the input.
*
* Ex:
* ```
* shallowEquals(
* sortOn (head, [[2, "world"], [4, "!"], [1, "Hello"]]),
* [[1,"Hello"],[2,"world"],[4,"!"]]
* ) // true
* ```
* @function module:list.sortOn
* @param valueFn {Function}
* @param xs {Array|String|*}
* @returns {Array}
*/
sortOn = curry((valueFn, xs) =>
// Un-decorate
map(decorated => decorated[1],
// Decorate and sort
sortBy(
// Ordering
([a0], [b0]) => genericAscOrdering(a0, b0),
// Decorate
map(item => [valueFn(item), item], xs)
)
)
),
/**
* The sortBy function is the non-overloaded (in haskell terms) version of sort.
* @haskellExample ```
* >>> sortBy (\(a,_) (b,_) -> compare a b) [(2, "world"), (4, "!"), (1, "Hello")]
* [(1,"Hello"),(2,"world"),(4,"!")]
* ```
* @function module:list.sortBy
* @param orderingFn {Function}
* @param xs {Array|String|*}
* @returns {Array|String|*}
*/
sortBy = curry((orderingFn, xs) => sliceCopy(xs).sort(orderingFn || genericAscOrdering)),
/**
* The insert function takes an element and a list and inserts the element
* into the list at the first position where it is less than or equal to the
* next element. In particular, if the list is sorted before the call, the
* result will also be sorted. It is a special case of insertBy, which allows
* the programmer to supply their own comparison function.
* @function module:list.insert
* @param x {*}
* @param xs {Array|*}
* @returns {Array}
*/
insert = curry((x, xs) => {
if (!xs.length) {
return of(xs, x);
}
const foundIndex = findIndex(item => x <= item, xs);
return foundIndex === -1 ? concat([xs, of(xs, x)]) :
concat(intersperse(of(xs, x), splitAt(foundIndex, xs)));
}),
/**
* A version of `insert` that allows you to specify the ordering of the inserted
* item; Before/at, or after
* @function module:list.insertBy
* @haskellType `insertBy :: (a -> a -> Ordering) -> a -> [a] -> [a]`
* @note `Ordering` means 'something that is order-able'
* operated on by this functions logic.
* @param orderingFn {Function} - A function that returns `-1`, `0`, or 1`.
* @param x {*} - Value to insert.
* @param xs {Array} - List to insert into (note new list is returned)
* @returns {Array} - New list.
*/
insertBy = curry((orderingFn, x, xs) => {
const limit = length(xs);
if (!limit) {
return [x];
}
let ind = 0;
for (; ind < limit; ind += 1) {
if (orderingFn(x, xs[ind]) <= 0) {
const parts = splitAt(ind, xs);
return concat([parts[0], [x], parts[1]]);
}
}
return aggregateArray(sliceCopy(xs), x);
}),
/**
* The nubBy function behaves just like nub, except it uses a user-supplied equality predicate.
* @function module:list.nubBy
* @param pred {Function}
* @param list {Array|String|*}
* @returns {Array}
*/
nubBy = curry((pred, list) => {
if (!length(list)) {
return [];
}
const limit = length(list);
let ind = 0,
currItem,
out = [],
anyOp = storedItem => pred(currItem, storedItem);
for (; ind < limit; ind += 1) {
currItem = list[ind];
if (any(anyOp, out)) {
continue;
}
out.push(currItem);
}
return out;
}),
/**
* Behaves the same as `remove`, but takes a user-supplied equality predicate.
* @function module:list.removeBy
* @param pred {Function} - Equality predicate `(a, b) => bool`
* @param x {*}
* @param list {Array|String|*}
* @returns {Array}
*/
removeBy = curry((pred, x, list) => {
const foundIndex = findIndex(item => pred(x, item), list);
if (foundIndex > -1) {
const parts = splitAt(foundIndex, list);
return append(parts[0], tail(parts[1]));
}
return sliceCopy(list);
}),
/**
* The `removeFirstsBy` function takes a predicate and two lists and returns the first list with the first
* occurrence of each element of the second list removed.
* @function module:list.removeFirstBy
* @param pred {Function}
* @param xs1 {Array|String|*}
* @param xs2 {Array|String|*}
* @returns {Array}
*/
removeFirstsBy = curry((pred, xs1, xs2) =>
foldl((agg, x) => removeBy(pred, x, agg), xs1, xs2)),
/**
* Returns the union on elements matching boolean check passed in.
* @function module:list.unionBy
* @param pred {Function} - `pred :: a -> a -> Bool`
* @param arr1 {Array}
* @param arr2 {Array}
* @returns {Array}
*/
unionBy = curry((pred, arr1, arr2) =>
foldl((agg, b) => {
const alreadyAdded = any(a => pred(a, b), agg);
return !alreadyAdded ? (agg.push(b), agg) : agg;
}, sliceCopy(arr1), arr2
)),
/**
* Creates a union on matching elements from array1.
* @function module:list.union
* @param arr1 {Array}
* @param arr2 {Array}
* @returns {Array}
*/
union = curry((arr1, arr2) =>
append(arr1,
filter(elm => !includes(elm, arr1), arr2))),
/**
* Performs an intersection on list 1 with elements from list 2.
* @function module:list.intersect
* @param arr1 {Array}
* @param arr2 {Array}
* @returns {Array}
*/
intersect = curry((arr1, arr2) =>
!arr1 || !arr2 || (!arr1 && !arr2) ? [] :
filter(elm => includes(elm, arr2), arr1)),
/**
* Returns an intersection by predicate.
* @function module:list.intersectBy
* @param pred {Function} - `pred :: a -> b -> Bool`
* @param list1 {Array}
* @param list2 {Array}
* @return {Array}
*/
intersectBy = curry((pred, list1, list2) =>
foldl((agg, a) =>
any(b => pred(a, b), list2) ? (agg.push(a), agg) : agg
, [], list1)),
/**
* Returns the difference of list 1 from list 2.
* @note The `difference` operation here is non-associative; E.g., `a - b` is not equal to `b - a`;
* @function module:list.difference
* @param array1 {Array}
* @param array2 {Array}
* @returns {Array}
*/
difference = curry((array1, array2) => { // augment this with max length and min length ordering on op
if (array1 && !array2) {
return sliceCopy(array1);
}
else if (!array1 && array2 || (!array1 && !array2)) {
return [];
}
return reduce((agg, elm) =>
!includes(elm, array2) ? (agg.push(elm), agg) : agg
, [], array1);
}),
/**
* Returns the complement of list 0 and the reset of the passed in arrays.
* @function module:list.complement
* @param arr0 {Array}
* @param arrays {...Array}
* @returns {Array}
*/
complement = curry2((arr0, ...arrays) =>
reduce((agg, arr) => append(agg, difference(arr, arr0)), [], arrays));


All stocks on NYSE and NASDAQ are used as the investment universe. We create a class to save all variables for each symbol.
class SymbolData: def __init__(self, symbol): self.symbol = symbol self.window = RollingWindow[float](13) self.GARR_ratio = None self.yearly_return = None
To get the value for the above variables, we save the stock price at the start of each month in the rolling window for the last 12 months and calculate the monthly return and the yearly return. Then we can compute the last month's geometric average rate of return (GARR) and the previous 12-month GARR. The formulas are as follows
\[GARR_{last\ 12\ months}=(1+R_1)^{1/12}\cdot(1+R_2)^{1/12}\cdot\cdot\cdot(1+R_{11})^{1/12}\cdot(1+R_{12})^{1/12}-1\] \[GARR_{last\ months}=(1+R_{last\ month})^{1/12}-1\]
We assign the value of two returns to each symbol in SymbolData
class. All stocks are sorted based on their past 12-month return.
Stocks are then divided into 3 portfolios (top 30% - winner group, middle 40% and bottom 30% - loser group). In the winner group, stocks are further
classified into two categories: the return-increasing winner and return-decreasing winner using the ratio of last month's geometric average rate of return (GARR) over the last 12 month GARR.
def CoarseSelectionFunction(self, coarse): if self.month_start: self.coarse = True coarse = [i for i in coarse if i.AdjustedPrice > 10] for i in coarse: if i.Symbol not in self.SymbolPrice: self.SymbolPrice[i.Symbol] = SymbolData(i.Symbol) self.SymbolPrice[i.Symbol].window.Add(float(i.AdjustedPrice)) if self.SymbolPrice[i.Symbol].window.IsReady: price = np.array([i for i in self.SymbolPrice[i.Symbol].window]) returns = (price[:-1]-price[1:])/price[1:] self.SymbolPrice[i.Symbol].yearly_return = (price[0]-price[-1])/price[-1] GARR_12 = np.prod([(1+i)**(1/12) for i in returns])-1 GARR_1 = (1+returns[0])**(1/12)-1 self.SymbolPrice[i.Symbol].GARR_ratio = GARR_1 / GARR_12
The decreasing-return winner group contains 13 stocks in winner group with the lowest GARR Ratio and vice-versa for the increasing-return winner group.
ReadySymbolPrice = {symbol: SymbolData for symbol, SymbolData in self.SymbolPrice.items() if SymbolData.window.IsReady} if ReadySymbolPrice and len(ReadySymbolPrice)>50: # sort stocks in coarse by last 12-month return sorted_by_return = sorted(ReadySymbolPrice, key = lambda x: ReadySymbolPrice[x].yearly_return) # top 30% with the highest 12-month return goes into winner group winner = sorted_by_return[:int(len(sorted_by_return)*0.3)] # bottom 30% with the lowest 12-month return goes into loser group loser = sorted_by_return[-int(len(sorted_by_return)*0.3):] self.decrease_winner = sorted(winner, key = lambda x: ReadySymbolPrice[x].GARR_ratio)[:15] self.increase_loser = sorted(loser, key = lambda x: ReadySymbolPrice[x].GARR_ratio)[-15:] return self.decrease_winner+self.increase_loser
The algorithm goes long stocks from the decreasing-return winner group and short stocks from the increasing-return loser group. The portfolio is created as equally weighted and rebalanced on a monthly basis.
def OnData(self, data): if self.month_start and self.coarse: self.month_start = False self.coarse = False if all([self.decrease_winner, self.increase_loser]): stocks_invested = [x.Key for x in self.Portfolio] for i in stocks_invested: if i not in self.decrease_winner+self.increase_loser: self.Liquidate(i) short_weight = 0.5/len(self.increase_loser) for j in self.increase_loser: self.SetHoldings(j, -short_weight) long_weight = 0.5/len(self.decrease_winner) for i in self.decrease_winner: self.SetHoldings(i, long_weight)


Jacob Hyman
Unafffiliated
Oncology Social Worker - U. C. Health
Bachelors Degree - Psychology Masters Degree - Social Work
80113
As a social worker, I have made it my mission to advocate for and represent the interest of marginalized groups in Colorado and around the world by challenging the status quo and working to gain equality for even the most oppressed and invisible among us.
Andrew Romanoff, Joe Biden/Kamala Harris, Bernie Sanders, Hillary Clinton, No on 115, ActionAid USA, Planned Parenthood, Movement for Black Lives, Yes on 114, Swing Left, Third Way Center, Colorado Coalition for the Homeless
National Association of Social Workers Third Way Center - Young Professionals Board
I believe that the work of this commission is essential to ensuring fairness in the way in which our legislature is districted so that no group's interests are being held above any other's.
As a former researcher and current medical social worker, I am faced with the necessity of impartiality and fairness every day. No one patient or client can be more important to me than another, and I am encouraged professionally and personally to constantly monitor my own biases and challenge them internally and in group/team discussions. I believe that honestly, frankly, and sometimes painfully addressing our own biases is the key to the change that is at the heart of this commission's work, and that is the perspective that I bring in order to promote consensus and agreement -- greater awareness of ourselves and, in turn, of each other.
I have published research multiple times throughout my career, and very much enjoy approaching problems logically and analytically. My research has focused on sociol-emotional learning, substance use in young adults, PTSD and Depression, and severe mental illness. I am able to look past the emotionality of intense issues in order to see the underlying systemic causes and effects -- from substance abuse and mental health, to economics, to abortion.
Male


w3schools Provides awesome templates under open license, the problem is these templates use w3css framework which actually is a nice one but not popular as Bootstrap so That's what we are aiming for.


I started writing that programming doesn't trigger any strong emotional responses for me apart from mild frustration. Then I remembered Code Academy, when my frustration started turning into anger and there was a lot of "console.log("Code Academy sucks")" going on.
In general though, it is much easier to control one's responses when engaging with a computer. Humans are trickier. I used the Siberian North Railroad exercise to mentally rehearse how I could have better responded to a situation that made me angry. At the time, my response was to remove myself from the situation physically and calm down. While this was somewhat effective, I still wished I could have been more compassionate in my response as I wasn't able to completely control my emotions and my anger was apparent. I think that if I had allowed myself a moment to observe/notice and reflect I might have been able to remain calmer and have a less emotional response to the situation.
As this is a stressful time of year and I tend to get very irritable when stressed, I'm going to have plenty of opportunities to practice the Siberian North Railroad technique. When it comes to programming, I think the compassionate response is self-directed and may quite often be a simple action such as taking a short break.


class: title # Docker
Orchestration
Workshop --- ## Outline (1/2) - Pre-requirements - VM environment - Our sample application - Running services independently - Running the whole app on a single node - Identifying bottlenecks - Measuring latency under load - Scaling HTTP on a single node - Put a load balancer on it - Connecting to containers on other hosts - Abstracting remote services with ambassadors - Various considerations about ambassadors --- ## Outline (2/2) - Docker for ops - Backups - Logs - Security upgrades - Network traffic analysis - Dynamic orchestration - Hands-on Swarm - Deploying Swarm - Cluster discovery - Building our app on Swarm - Network plumbing on Swarm - Going further --- # Pre-requirements - Computer with network connection and SSH client
(on Windows, get [putty](http://www.putty.org/) or [Git BASH](https://msysgit.github.io/)) - GitHub account (recommended; not mandatory) - Docker Hub account (only for Swarm hands-on section) - Basic Docker knowledge .exercise[ - This is the stuff you're supposed to do! - Create [GitHub](https://github.com/) and [Docker Hub](https://hub.docker.com) accounts now if needed - Go to [lisa.dckr.info](http://lisa.dckr.info) to view these slides ] --- # VM environment - Each person gets 5 VMs - They are *your* VMs - They'll be up until tomorrow - You have a little card with login+password+IP addresses - You can automatically SSH from one VM to another .exercise[ - Log into the first VM (`node1`) - Check that you can SSH (without password) to `node2` - Check the version of docker with `docker version` ] .footnote[Note: from now on, unless instructed, **all commands must be run from the first VM, `node1`**.] --- ## Brand new versions! - Engine 1.9.0 - Compose 1.5.0 - Swarm 1.0 - Machine 0.5.0 --- # Our sample application - Let's look at the general layout of the [source code](https://github.com/jpetazzo/orchestration-workshop) - Each directory = 1 microservice - `rng` = web service generating random bytes - `hasher` = web service computing hash of POSTed data - `worker` = background process using `rng` and `hasher` - `webui` = web interface to watch progress .exercise[ - Clone the repository on `node1`:
.small[`git clone git://github.com/jpetazzo/orchestration-workshop`] ] (Bonus points for forking on GitHub and cloning your fork!) --- ## What's this application? - It is a DockerCoin miner! 💰🐳📦🚢 - No, you can't buy coffee with DockerCoins - How DockerCoins works: - `worker` asks to `rng` to give it random bytes - `worker` feeds those random bytes into `hasher` - each hash starting with `0` is a DockerCoin - DockerCoins are stored in `redis` - `redis` is also updated every second to track speed - you can see the progress with the `webui` Next: we will inspect components independently. --- # Running services independently First, we will run the random number generator (`rng`). .exercise[ - Go to the `dockercoins` directory (in the cloned repo) - Run `docker-compose up rng`
(Docker will pull `python` and build the microservice) ] .icon[![Warning](warning.png)] Pay attention to the port mapping! - The container log says:
`Running on http://<IP_ADDRESS>:80/` - But if you try `curl localhost:80`, you will get:
`Connection refused` --- ## Understanding port mapping - `node1`, the Docker host, has only one port 80 - If we give the one and only port 80 to the first container who asks for it, we are in trouble when another container needs it - Default behavior: containers are not "exposed"
(only reachable through their private address) - Container network services can be exposed: - statically (you decide which host port to use) - dynamically (Docker allocates a host port) --- ## Declaring port mapping - Directly with the Docker Engine:
`docker run -P redis`
`docker run -p 6379 redis`
`docker run -p 1234:6379 redis` - With Docker Compose, in the `docker-compose.yml` file: ``` rng: … ports: - "8001:80" ``` → port 8001 *on the host* maps to port 80 *in the container* --- ## Using the `rng` service Let's get random bytes of data! .exercise[ - Open a second terminal and connect to the same VM - Check that the service is alive:
`curl localhost:8001` - Get 10 bytes of random data:
`curl localhost:8001/10` - If the binary data output messed up your terminal, fix it:
`reset` ] --- ## Running the hasher .exercise[ - Run `docker-compose up hasher`
(it will pull `ruby` and do the build) ] .icon[![Warning](warning.png)] Again, pay attention to the port mapping! The container log says that it's listening on port 80, but it's mapped to port 8002 on the host. You can see the mapping in `docker-compose.yml`. --- ## Testing the hasher .exercise[ - Open a third terminal window, and SSH to `node1` - Run `curl localhost:8002`
(it will say it's alive) - Posting binary data requires some extra flags: ``` curl \ -H "Content-type: application/octet-stream" \ --data-binary hello \ localhost:8002 ``` - Check that it computed the right hash:
`echo -n hello | sha256sum` ] --- ## Stopping services We have multiple options: - Interrupt `docker-compose up` with `^C` - Stop individual services with `docker-compose stop rng` - Stop all services with `docker-compose stop` - Kill all services with `docker-compose kill`
(rude, but faster!) .exercise[ - Use any of those methods to stop `rng` and `hasher` ] --- # Running the whole app on a single node .exercise[ - Run `docker-compose up` to start all components ] - Aggregate output is shown - Output is verbose
(because the worker is constantly hitting other services) - Now let's use the little web UI to see realtime progress .exercise[ - Open http://[yourVMaddr]:8000/ (from a browser) ] --- ## Running in the background - The logs are very verbose (and won't get better) - Let's put them in the background for now! .exercise[ - Stop the app (with `^C`) - Start it again with `docker-compose up -d` - Check on the web UI that the app is still making progress ] --- ## Looking at resource usage - Let's look at CPU, memory, and I/O usage .exercise[ - run `top` to see CPU and memory usage
(you should see idle cycles) - run `vmstat 3` to see I/O usage (si/so/bi/bo)
(the 4 numbers should be almost zero,
except `bo` for logging) ] We have available resources. - Why? - How can we use them? --- ## Scaling workers on a single node - Docker Compose supports scaling.red[*] - Let's scale `worker` and see what happens! .exercise[ - In one SSH session, run `docker-compose logs worker` - In another, run `docker-compose scale worker=10` - See the impact on CPU load (with top/htop),
and on compute speed (with web UI) ] You should see the compute speed increase ~3x (not 10x). .footnote[.red[*]With some limitations, as we'll see later.] --- # Identifying bottlenecks - Adding workers didn't result in linear improvement - *Something else* is slowing us down -- - ... But what? -- - The code doesn't have instrumentation - We will use `ab` for individual load testing - We will use `httping` to view latency under load --- ## Benchmarking our microservices We will test microservices in isolation. .exercise[ - Stop the application: `docker-compose kill` - Remove old containers: `docker-compose rm` - Start `hasher` and `rng`: `docker-compose up hasher rng` ] Now let's hammer them with requests! --- ## Testing `rng` Let's assess the raw performance of our RNG. .exercise[ - Test the performance on one big request:
`curl -o/dev/null localhost:8001/10000000`
(should take ~1s, and show speed of ~10 MB/s) ] If we were doing requests of 1000 bytes ... ... Could we get 10k req/s? Let's test and see what happens! --- ## Concurrent requests .exercise[ - Test 100 requests of 1000 bytes each:
`ab -n 100 localhost:8001/1000` - Test 100 requests, 10 requests in parallel:
`ab -n 100 -c 10 localhost:8001/1000`
(look how the latency has increased!) - Try with 100 requests in parallel:
`ab -n 100 -c 100 localhost:8001/1000` ] -- Whatever we do, we get ~10 requests/second. Increasing concurrency doesn't help: it just increases latency. --- ## Discussion - When serving requests sequentially, they each take 100ms - When 10 requests arrive at the same time: - one request is served in 100ms - another is served in 200ms - another is served in 300ms - ... - another is served in 1000ms - All requests are queued and served by a single thread - It looks like `rng` doesn't handle concurrent requests - What about `hasher`? --- ## Save some random data and stop the generator Before testing the hasher, let's save some random data that we will feed to the hasher later. .exercise[ - Run `curl localhost:8001/1000000 > /tmp/random` ] Now we can stop the generator. .exercise[ - In the shell where you did `docker-compose up rng`,
stop it by hitting `^C` ] --- ## Benchmarking the hasher We will hash the data that we just got from `rng`. .exercise[ - Posting binary data requires some extra flags: ``` curl \ -H "Content-type: application/octet-stream" \ --data-binary @/tmp/random \ localhost:8002 ``` - Compute the hash locally to verify that it works fine:
`sha256sum /tmp/random`
(it should display the same hash) ] --- ## The hasher under load The invocation of `ab` will be slightly more complex as well. .exercise[ - Execute 100 requests in a row: ``` ab -n 100 -T application/octet-stream \ -p /tmp/random localhost:8002/ ``` - Execute 100 requests with 10 requests in parallel: ``` ab -c 10 -n 100 -T application/octet-stream \ -p /tmp/random localhost:8002/ ``` ] Take note of the performance numbers (requests/s). --- ## Benchmarking the hasher on smaller data Here we hashed 1,000,000 bytes. Later we will hash much smaller payloads. Let's repeat the tests with smaller data. .exercise[ - Run `truncate --size=10 /tmp/random` - Repeat the `ab` tests ] --- # Measuring latency under load We will use `httping`. .exercise[ - You need three SSH connections - In the first one, let run `httping localhost:8001` - In the second one, let run `httping localhost:8002` - In the third one, run `docker-compose up -d` ] Check the latency numbers. - `hasher` should be very low (~1ms) - `rng` should be low, with occasional spikes (10-100ms) --- ## Latency when scaling the worker We will add workers and see what happens. .exercise[ - Run `docker-compose scale worker=2` - Check latency - Increase number of workers and repeat ] What happens? - `hasher` remains low - `rng` spikes up until it is reaches ~N*100ms
(when you have N+1 workers) --- class: title Why? --- ## Why does everything take (at least) 100ms? -- `rng` code: ![RNG code screenshot](delay-rng.png) -- `hasher` code: ![HASHER code screenshot](delay-hasher.png) --- class: title But ... WHY?!? --- ## Why did we sprinkle this sample app with sleeps? - Deterministic performance
(regardless of instance speed, CPUs, I/O...) -- - Actual code sleeps all the time anyway -- - When your code makes a remote API call: - it sends a request; - it sleeps until it gets the response; - it processes the response. --- ## Why do `rng` and `hasher` behave differently? ![Equations on a blackboard](equations.png) -- (Synchronous vs. asynchronous event processing) --- ## How to make `rng` go faster - Obvious solution: comment out the `sleep` instruction -- - Real-world solution: use an asynchronous framework
(e.g. use gunicorn with gevent) -- - New rule: we can't change the code! -- - Solution: scale out `rng`
(dispatch `rng` requests on multiple instances) --- # Scaling HTTP on a single node - We could try to scale with Compose: ``` docker-compose scale rng=3 ``` - Compose doesn't deal with load balancing - We would get 3 instances ... - ... But only the first one would serve traffic --- ## The plan - Stop the `rng` service first - Create multiple identical `rng` containers - Put a load balancer in front of them - Point other services to the load balancer --- ## Stopping `rng` - That's the easy part! .exercise[ - Use `docker-compose` to stop `rng`: ``` docker-compose stop rng ``` ] Note: we do this first because we are about to remove `rng` from the Docker Compose file. If we don't stop `rng` now, it will remain up and running, with Compose being unaware of its existence! --- ## Scaling `rng` .exercise[ - Replace the `rng` service with multiple copies of it: ``` rng1: build: rng rng2: build: rng rng3: build: rng ``` ] That's all! Shortcut: `docker-compose.yml-scaled-rng` --- ## Introduction to `jpetazzo/hamba` - Public image on the Docker Hub - Load balancer based on HAProxy - Expects the following arguments:
`FE-port BE1-addr BE1-port BE2-addr BE2-port ...`
*or*
`FE-addr:FE-port BE1-addr BE1-port BE2-addr BE2-port ...` - FE=frontend (the thing other services connect to) - BE=backend (the multiple copies of your scaled service) .small[ Example: listen to port 80 and balance traffic on www1:1234 + www2:2345 ``` docker run -d -p 80 jpetazzo/hamba 80 www1 1234 www2 2345 ``` ] --- # Put a load balancer on it Let's add our load balancer to the Compose file. .exercise[ - Add the following section to the Compose file: ``` rng0: image: jpetazzo/hamba links: - rng1 - rng2 - rng3 command: 80 rng1 80 rng2 80 rng3 80 ports: - "8001:80" ``` ] Shortcut: `docker-compose.yml-scaled-rng` --- ## Point other services to the load balancer - The only affected service is `worker` - We have to replace the `rng` link with a link to `rng0`, but it should still be named `rng` (so we don't change the code) .exercise[ - Update the `worker` section as follows: ``` worker: build: worker links: - rng0:rng - hasher - redis ``` ] Shortcut: `docker-compose.yml-scaled-rng` --- ## Start the whole stack .exercise[ - Run `docker-compose up -d` - Check worker logs with `docker-compose logs worker` - Check load balancer logs with `docker-compose logs rng0` ] If you get errors about port 8001, make sure that `rng` was stopped correctly and try again. --- ## The good, the bad, the ugly - The good We scaled a service, added a load balancer -
without changing a single line of code. - The bad We manually copy-pasted sections in `docker-compose.yml`. Improvement: write scripts to transform the YAML file. - The ugly If we scale up/down, we have to restart everything. Improvement: reconfigure the load balancer dynamically. --- # Connecting to containers on other hosts - So far, our whole stack is on a single machine - We want to scale out (across multiple nodes) - We will deploy the same stack multiple times - But we want every stack to use the same Redis
(in other words: Redis is our only *stateful* service here) -- - And remember: we're not allowed to change the code! - the code connects to host `redis` - `redis` must resolve to the address of our Redis service - the Redis service must listen on the default port (6379) --- ## The plan - Deploy our Redis service separately - use the same `redis` image - make sure that Redis server port (6379) is publicly accessible, using port 6379 on the Docker host - Update our Docker Compose YAML file - remove the `redis` section - in the `links` section, remove `redis` - instead, put a `redis` entry in `extra_hosts` Note: the code stays on the first node!
(We do not need to copy the code to the other nodes.) --- ## Making Redis available on its default port There are two strategies. - `docker run -p 6379:6379 redis` - the container has its own, isolated network stack - Docker creates a port mapping rule through iptables - slight performance overhead - port number is explicit (visible through Docker API) - `docker run --net host redis` - the container uses the network stack of the host - when it binds to 6379/tcp, that's 6379/tcp on the host - allows raw speed (no overhead due to iptables/bridge) - port number is not visible through Docker API Choose wisely! --- ## Deploy Redis .exercise[ - Start a new redis container, mapping port 6379 to 6379: ``` docker run -d -p 6379:6379 redis ``` - Check that it's running with `docker ps` - Note the IP address of this Docker host - Try to connect to it (from anywhere): ``` telnet ip.ad.dr.ess 6379 ``` ] To exit a telnet session: `Ctrl-] c ENTER` --- ## Update `docker-compose.yml` (1/3) .exercise[ - Comment out `redis`: ``` #redis: # image: redis ``` ] --- ## Update `docker-compose.yml` (2/3) .exercise[ - Update `worker`: ``` worker: build: worker extra_hosts: redis: A.B.C.D links: - rng0:rng - hasher ``` ] Replace `A.B.C.D` with the IP address noted earlier. Shortcut: `docker-compose.yml-extra-hosts`
(But you still have to replace `A.B.C.D`!) --- ## Update `docker-compose.yml` (3/3) .exercise[ - Update `webui`: ``` webui: build: webui extra_hosts: redis: A.B.C.D ports: - "8000:80" #volumes: # - "./webui/files/:/files/" ``` ] (Replace `A.B.C.D` with the IP address noted earlier) .icon[![Warning](warning.png)] Don't forget to comment out the `volumes` section! --- ## Why did we comment out the `volumes` section? - Volumes have multiple uses: - storing persistent stuff (database files...) - sharing files between containers (logs, configuration...) - sharing files between host and containers (source...) - The `volumes` directive expands to an host path
.small[(e.g. `/home/docker/orchestration-workshop/dockercoins/webui/files`)] - This host path exists on the local machine
(not on the others) - This specific volume is used in development
(not in production) --- ## Start the stack on the first machine - Nothing special to do here - Just bring up the application like we did before .exercise[ - `docker-compose up -d` ] - Check in the web browser that it's running correctly --- ## Start the stack on another machine - We will set the `DOCKER_HOST` variable - `docker-compose` will detect and use it - Our Docker hosts are listening on port 55555 .exercise[ - Set the environment variable:
`export DOCKER_HOST=tcp://node2:55555` - Start the stack:
`docker-compose up -d` - Check that it's running:
`docker-compose ps` ] --- ## Scale! .exercise[ - Open the Web UI
(on a node where it's deployed) - Deploy one instance of the stack on each node ] --- ## Cleanup - Let's remove what we did .exercise[ - You can use the following scriptlet: ``` for N in $(seq 1 5); do export DOCKER_HOST=tcp://node$N:55555 docker ps -qa | xargs docker rm -f done unset DOCKER_HOST ``` ] --- # Abstracting remote services with ambassadors - What if we can't/won't run Redis on its default port? - What if we want to be able to move it more easily? -- - We will use an ambassador - Redis will run at an arbitrary location (host+port) - The ambassador will be part of the scaled stack - The ambassador will connect to Redis - The ambassador will "act as" Redis in the stack --- ## Start redis - This time, we will let Docker pick the port for Redis .exercise[ - Run redis with a random public port:
`docker run -d -P --name myredis redis` - Check which port was allocated:
`docker port myredis 6379` ] - Note this IP address and port --- ## Update `docker-compose.yml` .exercise[ - Restore `links` as they were before in `webui` and `worker` - Replace `redis` with an ambassador using `jpetazzo/hamba`: ``` redis: image: jpetazzo/hamba command: 6379 AA.BB.CC.DD EEEEE ``` ] Shortcut: `docker-compose.yml-ambassador`
(But you still have to update `AA.BB.CC.DD EEEE`!) --- ## Start the new stack .exercise[ - Run `docker-compose up -d` - Go to the web UI - Start the stack on another node as previously,
and confirm on the web UI that it's picking up ] --- # Various considerations about ambassadors - "But, ambassadors are adding an extra hop!" -- - Yes, but if you need load balancing, you need that hop - Ambassadors actually *save* one hop
(they act as local load balancers) - traditional load balancer:
client ⇒ external LB ⇒ server (2 physical hops) - ambassadors:
client → ambassador ⇒ server (1 physical hop) -- - Ambassadors are more reliable than traditional LBs
(they are colocated with their clients) --- ## Inconvenients of ambassadors - Generic issues
(shared with any kind of load balancing / HA setup) - extra logical hop (not transparent to the client) - must assess backend health - one more thing to worry about (!) - Specific issues - load balancing fairness High-end load balancing solutions will rely on back pressure from the backends. This addresses the fairness issue. --- ## There are many ways to deploy ambassadors "Ambassador" is a design pattern. There are many ways to implement it. We will present three increasingly complex (but also powerful) ways to deploy ambassadors. --- ## Single-tier ambassador deployment - One-shot configuration process - Must be executed manually after each scaling operation - Scans current state, updates load balancer configuration - Pros:
- simple, robust, no extra moving part
- easy to customize (thanks to simple design)
- can deal efficiently with large changes - Cons:
- must be executed after each scaling operation
- harder to compose different strategies - Example: this workshop --- ## Two-tier ambassador deployment - Daemon listens to Docker events API - Reacts to container start/stop events - Adds/removes back-ends to load balancers configuration - Pros:
- no extra step required when scaling up/down - Cons:
- extra process to run and maintain
- deals with one event at a time (ordering matters) - Hidden gotcha: load balancer creation - Example: interlock --- ## Three-tier ambassador deployment - Daemon listens to Docker events API - Reacts to container start/stop events - Adds/removes scaled services in distributed config DB
(zookeeper, etcd, consul…) - Another daemon listens to config DB events - Adds/removes backends to load balancers configuration - Pros:
- more flexibility - Cons:
- three extra services to run and maintain - Example: registrator --- ## Other multi-host communication mechanisms - Overlay networks - weave, flannel, pipework ... - Network plugins - available since Engine 1.9 - Allow a flat network for your containers - Often requires an extra service to deal with BUM packets
(broadcast/unknown/multicast) - e.g. a key/value store (Consul, Etcd, Zookeeper ...) - Load balancers and/or failover mechanisms still needed --- class: title # Interlude
# Docker for ops --- # Backups - Redis is still running (with name `myredis`) - We want to enable backups without touching it - We will use a special backup container: - sharing the same volumes - linked to it (to connect to it easily) - possibly containing our backup tools - This works because the `redis` container image
stores its data on a volume --- ## Starting the backup container .exercise[ - Start the container: ``` docker run --link myredis:redis \ --volumes-from myredis \ -v /tmp/myredis:/output \ -ti ubuntu ``` - Look in `/data` in the container
(That's where Redis puts its data dumps) ] - We need to tell Redis to perform a data dump *now* --- ## Connecting to Redis .exercise[ - `apt-get install telnet` - `telnet redis 6379` - issue `SAVE` then `QUIT` - Look at `/data` again ] - There should be a recent dump file now --- ## Getting the dump out of the container - We could use many things: - s3cmd to copy to S3 - SSH to copy to a remote host - gzip/bzip/etc before copying - We'll just copy it to the Docker host .exercise[ - Copy the file from `/data` to `/output` - Exit the container - Look into `/tmp/myredis` (on the host) ] --- # Logs - Sorry, this part won't be hands-on - Two strategies: - log to plain files on volumes - log to stdout
(and use a logging driver) --- ## Logging to plain files on volumes - Start a container with `-v /logs` - Make sure that all log files are in `/logs` - To check logs, run e.g. ``` docker run --volumes-from ... ubuntu sh -c \ "grep WARN /logs/*.log" ``` - Or just go interactive: ``` docker run --volumes-from ... -ti ubuntu ``` - You can (should) start a log shipper that way --- ## Logging to stdout - All containers should write to stdout/stderr - Docker will collect logs and pass them to a logging driver - Available drivers:
json-file (default), syslog, journald, gelf, fluentd - Change driver by passing `--log-driver` option to daemon
(On Ubuntu, tweak `DOCKER_OPTS` in `/etc/default/docker`) - For now, only json-files supports logs retrieval
(i.e. `docker logs`) - Warning: json-file doesn't rotate logs by default
(but this can be changed with `--log-opt`) See: https://docs.docker.com/reference/logging/overview/ --- # Security upgrades - This section is not hands-on - Public Service Announcement - We'll discuss: - how to upgrade the Docker daemon - how to upgrade container images --- ## Upgrading the Docker daemon - Stop all containers cleanly
(`docker ps -q | xargs docker stop`) - Stop the Docker daemon - Upgrade the Docker daemon - Start the Docker daemon - Start all containers - This is like upgrading your Linux kernel,
but it will get better --- ## Upgrading container images - When a vulnerability is announced: - if it affects your base images,
make sure they are fixed first - if it affects downloaded packages,
make sure they are fixed first - re-pull base images - rebuild - restart containers (The procedure is simple and plain, just follow it!) --- # Network traffic analysis - We still have `myredis` running - We will use *shared network namespaces*
to perform network analysis - Two containers sharing the same network namespace... - have the same IP addresses - have the same network interfaces - `eth0` is therefore the same in both containers --- ## Install and start `ngrep` Ngrep uses libpcap (like tcpdump) to sniff network traffic. .exercise[ - Start a container with the same network namespace:
`docker run --net container:myredis -ti alpine` - Install ngrep:
`apk update && apk add ngrep` - Run ngrep:
`ngrep -tpd eth0 -Wbyline . tcp` ] You should see a stream of Redis requests and responses. --- class: title # Dynamic orchestration --- ## Static vs Dynamic - Static - you decide what goes where - simple to describe and implement - seems easy at first but doesn't scale efficiently - Dynamic - the system decides what goes where - requires extra components (HA KV...) - scaling can be finer-grained, more efficient --- ## Mesos (overview) - First presented in 2009 - Initial goal: resource scheduler
(two-level/pessimistic) - top-level "master" knows the global cluster state - "slave" nodes report status and resources to master - master allocates resources to "frameworks" - Container support added recently
(had to fit existing model) - Network and service discovery is complex --- ## Mesos (in practice) - Easy to setup a test cluster (in containers!) - Great to accommodate mixed workloads
(see Marathon, Chronos, Aurora, and many more) - "Meh" if you only want to run Docker containers - In production on clusters of thousands of nodes - Open source project; commercial support available --- ## Kubernetes (overview) - 1 year old - Designed specifically as a platform for containers
("greenfield" design) - "pods" = groups of containers sharing network/storage - Scaling and HA managed by "replication controllers" - extensive use of "tags" instead of e.g. tree hierarchy - Initially designed around Docker,
but doesn't hesitate to diverge in a few places --- ## Kubernetes (in practice) - Network and service discovery is powerful, but complex
.small[(different mechanisms within pod, between pods, for inbound traffic...)] - Initially designed around GCE
.small[(currently relies on "native" features for fast networking and persistence)] - Adaptation is needed when it differs from Docker
.small[(need to learn new API, new tooling, new concepts)] - Great deployment platform ...
but no developer experience yet Note: this slide probably needs an update, since KubeCon is happening *right now* in San Francisco! --- ## Swarm (in theory) - Consolidates multiple Docker hosts into a single one - "Looks like" a Docker daemon, but it dispatches (schedules) your containers on multiple daemons - Talks the Docker API front and back
(leverages the Docker API and ecosystem) - Open source and written in Go (like Docker) - Started by two of the original Docker authors
([@aluzzardi](https://twitter.com/aluzzardi) and [@vieux](https://twitter.com/vieux)) --- ## Swarm (in practice) - Stable since November 2015 - Great for some scenarios (Jenkins, grid...) - Requires extra effort for Compose build, links... - Requires a key/value store to achieve high availability - We'll see it (briefly) in action --- ## PAAS on Docker - The PAAS workflow: *just push code*
(inspired by Heroku, dotCloud...) - TL,DR: easier for devs, harder for ops,
some very opinionated choices - A few examples:
(Non-exhaustive list!!!) - Cloud Foundry - Deis - Dokku - Flynn - Tsuru --- ## A few other tools - Volume plugins (Convoy, Flocker...) - manage/migrate stateful containers (and more) - Network plugins (Contiv, Weave...) - overlay network so that containers can ping each other - Powerstrip - sits in front of the Docker API; great for experiments - Tutum - dashboard to manage feelts of Docker hosts ... And many more! ~~~ class: pic ![Here Be Dragons](dragons.jpg) ~~~ ## Warning: here be dragons - So far, we've used stable products (versions 1.X) - We're going to expore experimental software - **Use at your own risk** --- # Hands-on Swarm ![Swarm Logo](swarm.png) --- ## Setting up our Swarm cluster - This can be done manually or with **Docker Machine** - Manual deployment: - with TLS: certificate generation is painful
(needs dual-use certs) - without TLS: easier, but insecure
(unless you run on your internal/private network) - Docker Machine deployment: - generates keys, certificates, and deploys them for you - can also create VMs --- ## The Way Of The Machine - Install `docker-machine` (single binary download) - Set a few environment variables (cloud credentials) - Create one or more machines:
`docker-machine create -d digitalocean node42` - List machines and their status:
`docker-machine ls` - Select a machine for use:
`eval $(docker-machine env node42)`
(this will set a few environment variables) - Execute regular commands with Docker, Compose, etc.
(they will pick up remote host address from environment) --- ## Docker Machine `generic` driver - Most drivers work the same way: - use cloud API to create instance - connect to instance over SSH - install Docker - The `generic` driver skips the first step - It can install Docker on any machine,
as long as you have SSH access - We will use that! --- # Deploying Swarm - Components involved: - service discovery mechanism
(we'll use Docker's hosted system) - swarm manager
(runs on `node1`, exposes Docker API) - swarm agent
(runs on each node, registers it with service discovery) --- # Cluster discovery - Possible backends: - dynamic, self-hosted (zk, etcd, consul) - static (command-line or file) - hosted by Docker (token) - We will use the token mechanism .exercise[ - Run `TOKEN=$(docker run swarm create)` - Save `$TOKEN` carefully: it's your token
(it's the unique identifier for your cluster) ] --- ## Swarm agent - Used only for dynamic discovery (zk, etcd, consul, token) - Must run on each node - Every 20s (by default), tells to the discovery system: "Hello, there is a Swarm node at A.B.C.D:EFGH" - Must know the node's IP address
(sorry, it can't figure it out by itself, because
it doesn't know whether to use public or private addresses) - The node continues to work even if the agent dies - Automatically started by Docker Machine
(when the `--swarm` option is passed) --- ## Swarm manager - Today: must run on the leader node - Later: can run on multiple nodes, with leader election - Automatically started by Docker Machine
(when the `--swarm-master` option is passed) .exercise[ - Connect to `node1` - "Create" a node with Docker Machine .small[ ``` docker-machine create --driver generic \ --swarm --swarm-master --swarm-discovery token://$TOKEN \ --generic-ssh-user docker --generic-ip-address <IP_ADDRESS> node1 ``` ] ] (Don't forget to replace <IP_ADDRESS> with the node IP address!) --- ## Check our node Let's connect to the node *individually*. .exercise[ - Select the node with Machine ``` eval $(docker-machine env node1) ``` - Execute some Docker commands ``` docker version docker info docker ps ``` ] Two containers should show up: the agent and the manager. --- ## Check our (single-node) Swarm cluster Let's connect to the manager instead. .exercise[ - Select the Swarm manager with Machine ``` eval $(docker-machine env node1 --swarm) ``` - Execute some Docker commands ``` docker version docker info docker ps ``` ] The output is different! Let's review this. --- ## `docker version` Swarm identifies itself clearly: ``` Client: Version: 1.8.2 API version: 1.20 Go version: go1.4.2 Git commit: 0a8c2e3 Built: Thu Sep 10 19:19:00 UTC 2015 OS/Arch: linux/amd64 Server: Version: swarm/0.4.0 API version: 1.16 Go version: go1.4.2 Git commit: d647d82 Built: OS/Arch: linux/amd64 ``` --- ## `docker info` Swarm gives cluster information, showing all nodes: ``` Containers: 3 Images: 6 Role: primary Strategy: spread Filters: affinity, health, constraint, port, dependency Nodes: 1 node: <IP_ADDRESS>:2376 └ Containers: 3 └ Reserved CPUs: 0 / 2 └ Reserved Memory: 0 B / 3.86 GiB └ Labels: executiondriver=native-0.2, kernelversion=3.13.0-53-generic, operatingsystem=Ubuntu 14.04.2 LTS, provider=generic, storagedriver=aufs CPUs: 2 Total Memory: 3.86 GiB Name: 2ec2e6c4054e ``` --- ## `docker ps` - This one should show nothing at this point. - The Swarm containers are hidden. - This avoids unneeded pollution. - This also avoids killing them by mistake. --- ## Add other nodes to the cluster - Let's use *almost* the same command line
(but without `--swarm-master`) .exercise[ - Stay on `node1` (it has keys and certificates now!) - Add another node with Docker Machine .small[ ``` docker-machine create --driver generic \ --swarm --swarm-discovery token://$TOKEN \ --generic-ssh-user docker --generic-ip-address <IP_ADDRESS> node2 ``` ] ] Remember to update the IP address correctly. Repeat for all 4 nodes. Pro tip: look for name/address mapping in `/etc/hosts`. --- ## Scripting To help you a little bit: ``` grep node[2345] /etc/hosts | grep -v ^127 | while read IPADDR NODENAME do docker-machine create --driver generic \ --swarm --swarm-discovery token://$TOKEN \ --generic-ssh-user docker \ --generic-ip-address $IPADDR $NODENAME \ --- ## Running containers on Swarm Try to run a few `busybox` containers. Then, let's get serious: .exercise[ - Start a Redis service:
`docker run -dP redis` - See the service address:
`docker port $(docker ps -lq) 6379` ] This can be any of your five nodes. --- # Building our app on Swarm - Swarm has partial support for builds - .icon[![Warning](warning.png)] Older versions of Compose would crash on builds - Try it! .exercise[ - Run `docker-compose build` once ... - Run `docker-compose build` twice ... - What happened? ] --- ## Re-thinking the build process - Let's step back and think for a minute ... - What should `docker build` do on Swarm? - build on one machine - build everywhere ($$$) - After the build, what should `docker run` do? - run where we built (how do we know where it is?) - run on any machine that has the image - What do, what do‽ --- ## The plan - Build locally - Tag images - Upload them to the hub - Update the Compose file to use those images *That's the purpose of the `build-tag-push.py` script!* --- ## Build, Tag, And Push Let's inspect the source code of `build-tag-push.py` and run it. .icon[![Warning](warning.png)] It is better to run it against a single node! (There are some race conditions within Swarm when building+pushing too fast.) .exercise[ - Point to a single node:
`eval $(docker-machine env node1)` - Run the script (from the `dockercoins` directory):
`../build-tag-push.py` - Inspect the `docker-compose.yml-XXX` file that it created ] --- ## Can we run this now? Let's try! .exercise[ - Switch back to the Swarm cluster:
`eval $(docker-machine env node1 --swarm)` - Bring up the application:
`docker-compose -f docker-compose.yml-XXX up` ] -- It won't work, because Compose and Swarm do not collaborate to establish *placement constraints*. -- (╯°□°)╯︵ ┻━┻ --- ## Simple container dependencies - Container A has a link to container B - Compose starts B first, then A - Swarm translates the link into a placement constraint: - *"put A on the same node as B"* - Alles gut --- ## Complex container dependencies - Container A has a link to containers B and C - Compose starts B and C first
(but that can be on different nodes!) - Compose starts A - Swarm translates the links into placements contraints - *"put A on the same node as B"* - *"put A on the same node as C"* - If B and C are on different nodes, that's impossible So, what do‽ --- ## A word on placement constraints - Swarm supports constraints - We could tell swarm to put all our containers together - Linking would work - But all containers would end up on the same node -- - So having a cluster would be pointless! --- # Network plumbing on Swarm - We will use one-tier, dynamic ambassadors
(as seen before) - Other available options: - injecting service addresses in environment variables - implementing service discovery in the application - use Docker Engine Experimental + network plugins
(or any other overlay network like Weave or Pipework) --- ## Revisiting `jpetazzo/hamba` - Configuration is stored in a *volume* - A watcher process looks for configuration updates,
and restarts HAProxy when needed - It can be started without configuration: ``` docker run --name amba jpetazzo/hamba run ``` - There is a helper to inject a new configuration: ``` docker run --rm --volumes-from amba jpetazzo/hamba \ reconfigure 80 backend1 port1 backend2 port2 ... ``` .footnote[Note: configuration validation and error messages will be logged by the ambassador, not the `reconfigure` container.] --- ## Should we use `links` for our ambassadors? Technically, we could use links. - Before starting an app container: start the ambassador(s) it needs - When starting an app container: link it to its ambassador(s) But we wouldn't be able to use `docker-compose scale` anymore. --- ## Network namespaces and `extra_hosts` This is our plan: - Replace each `link` with an `extra_host`,
pointing to the `127.127.X.X` address space - Start app containers normally
(`docker-compose up`, `docker-compose scale`) - Start ambassadors after app containers are up: - ambassadors bind to `127.127.X.X` - they share their client's network namespace - Reconfigure ambassadors each time something changes --- ## Our plan for service discovery - Replace all `links` with static `/etc/hosts` entries - Those entries will map to `127.127.0.X`
(with different `X` for each service) - Example: `redis` will point to `<IP_ADDRESS>`
(instead of a container address) - Start all services; scale them if we want
(at this point, they will all fail to connect) - Start ambassadors in the services' namespace;
each ambassador will listen on the right `127.127.0.X` - Gather all backend addresses and configure ambassadors .icon[![Warning](warning.png)] Services should try to reconnect! --- ## "Design for failure," they said - When the containers are started, the network is not ready - First connection attempts **will fail** - App should try to reconnect - It is OK to crash and restart - Exponential back-off is nice --- ## Our tools - `link-to-ambassadors.py` - replaces all `links` with `extra_hosts` entries - `create-ambassadors.py` - scans running containers - allocates `127.127.X.X` addresses - starts (unconfigured) ambassadors - `configure-ambassadors.py` - scans running containers - gathers backend addresses - sends configuration to ambassadors --- ## Convert links to ambassadors - When we ran `build-tag-push.py` earlier,
it generated a new `docker-compose.yml-XXX` file. .exercise[ - Run the first script to create a new YAML file:
`../link-to-ambassadors.py docker-compose.yml-XXX a.yml` - Look how the file was modified:
`diff docker-compose.yml-XXX a.yml` ] The script can take one or two file name arguments: - two arguments indicate input and output files to use; - with one argument, the file will be modified in place. --- ## Bring up the application The application can now be started and scaled. Remember to use the *new* YAML file! .exercise[ - Start the application:
`docker-compose -f a.yml up -d` - Scale the application:
`docker-compose -f a.yml scale worker=5 rng=10` ] Note: you can scale everything as you like, *except Redis*, because it is stateful. --- ## Create the ambassadors This has to be executed each time you create new services or scale up existing ones. The script takes the YAML file as its only argument. It will scan and compare: - the list of app containers, - the list of ambassadors. It will create missing ambassadors. .exercise[ - Run the script!
`../create-ambassadors.py a.yml` ] --- ## Configure the ambassadors All ambassadors are created but they still need configuration. That's the purpose of the last script. It will gather: - the list of app backends, - the list of ambassadors. Then it configures all ambassadors with all found backends. .exercise[ - Run it!
`../configure-ambassadors.py a.yml` ] --- ## Check what we did .exercise[ - Find out the address of the web UI:
`docker-compose ps webui` - Point your browser to it - Check the logs:
`docker-compose logs` ] --- # Going further Scaling the application (difficulty: easy) - Run `docker-compose scale` - Re-create ambassadors - Re-configure ambassadors - No downtime --- ## Going further Deploying a new version (difficulty: easy) - Just re-run all the steps! - However, Compose will re-create the containers - You will have to re-create ambassadors
(and configure them) - You will have to cleanup old ambassadors
(left as an exercise for the reader) - You will experience a little bit of downtime --- ## Going further Zero-downtime deployment (difficulty: medium) - Isolate stateful services
(like we did earlier for Redis) - Do blue/green deployment: - deploy and scale version N - point a "top-level" load balancer to the app - deploy and scale version N+1 - put both apps in the "top-level" balancer - slowly switch traffic over to app version N+1 --- ## Going further Use the new networking features (difficulty: medium) - Create a key/value store (e.g. Consul cluster) - Reconfigure all Engines to use the key/value store - Load balancers can use DNS for backend discovery Note: this is really easy to do with a 1-node Consul cluster. --- ## Going further Harder projects: - Two-tier or three-tier ambassador deployments - Deploy to Mesos or Kubernetes --- class: title # Thanks!
Questions? ### [@jpetazzo](https://twitter.com/jpetazzo)
[@docker](https://twitter.com/docker)


We're here to take care of your requirements for raw materials, fine chemicals, intermediates for Pharma, Agro, Dyestuff, Paint, Aromatic & Cosmetic Industries.
We handle the entire hassle for your procurement.
For your import requirements or in case of any further queries, kindly email us at
<EMAIL_ADDRESS>or call us at
+91<PHONE_NUMBER>
+91<PHONE_NUMBER>
| Day | Time |
|---|---|
| Monday | 10:00 AM - 06:00 PM |
| Tuesday | 10:00 AM - 06:00 PM |
| Wednesday | 10:00 AM - 06:00 PM |
| Thursday | 10:00 AM - 06:00 PM |
| Friday | 10:00 AM - 06:00 PM |
| Saturday | 10:00 AM - 02:00 PM |
| Sunday | Closed |
Want to work with us?
Looking for candidates with minimum 1 year experience in the field and basic industry knowledge.
Send us an email at
leonachemicals@gmail.com


{% extends 'base.html' %} {% block content %}
{{ request.site.name }} is where you write loglines with instant feedback.
Write your first logline
Some loglines:


Tearshees act as a universal companion icon to upcoming Meetups. They contain the date of the event, but should never be relied on as the only conveyance of the event date.
.tearsheet-25
, .tearsheet
, .tearsheet-80
<time class="tearsheet-25">
Oct
25
</time>
These labels provide a standard way to indicate member status or role
.pill-label
, .pill-label--hollow
, .pill-label--leader
, .pill-label--friend
, .pill-label--paid
,.pill-label--unpaid
label label friend organizer paid unpaid

label
label
friend
organizer
paid
unpaid

.nametag-photo
, .nametag-photo-name

Takashi

Has a square photo

Figure sets are a common visual pattern on Meetup that consist of a figure (ie. an image) to the left of some descriptive text.
.figureset-right
is a variation of .figureset
that puts the figure on the right.
Food truck vinyl aute irony, umami terry richardson post-ironic ut id stumptown.
Cardigan consectetur cillum mollit accusamus letterpress. ...

This is a title

Food truck vinyl aute irony, umami terry richardson post-ironic ut id stumptown.

This is a title

Cardigan consectetur cillum mollit accusamus letterpress. ...


The fab tool is very verbose by default and prints out almost everything it can, including the remote end’s stderr and stdout streams, the command strings being executed, and so forth. While this is necessary in many cases in order to know just what’s going on, any nontrivial Fabric task will quickly become difficult to follow as it runs.
To aid in organizing task output, Fabric output is grouped into a number of non-overlapping levels or groups, each of which may be turned on or off independently. This provides flexible control over what is displayed to the user.
ノート
All levels, save for debug, are on by default.
The standard, atomic output levels/groups are as follows:
バージョン 0.9.2 で変更: Added “Executing task” lines to the running output level.
バージョン 0.9.2 で変更: Added the user output level.
There is a final atomic output level, debug, which behaves slightly differently from the rest:
debug: Turn on debugging (which is off by default.) Currently, this is largely used to view the “full” commands being run; take for example this run call:
run('ls "/home/username/Folder Name With Spaces/"')
Normally, the running line will show exactly what is passed into run, like so:
[hostname] run: ls "/home/username/Folder Name With Spaces/"
With debug on, and assuming you’ve left shell set to True, you will see the literal, full string as passed to the remote server:
[hostname] run: /bin/bash -l -c "ls \"/home/username/Folder Name With Spaces\""
Enabling debug output will also display full Python tracebacks during aborts.
ノート
Where modifying other pieces of output (such as in the above example where it modifies the ‘running’ line to show the shell and any escape characters), this setting takes precedence over the others; so if running is False but debug is True, you will still be shown the ‘running’ line in its debugging form.
バージョン 1.0 で変更: Debug output now includes full Python tracebacks during aborts.
In addition to the atomic/standalone levels above, Fabric also provides a couple of convenience aliases which map to multiple other levels. These may be referenced anywhere the other levels are referenced, and will effectively toggle all of the levels they are mapped to.
バージョン 1.4 で変更: Added the commands output alias.
You may toggle any of Fabric’s output levels in a number of ways; for examples, please see the API docs linked in each bullet point:
Direct modification of fabric.state.output: fabric.state.output is a dictionary subclass (similar to env) whose keys are the output level names, and whose values are either True (show that particular type of output) or False (hide it.)
fabric.state.output is the lowest-level implementation of output levels and is what Fabric’s internals reference when deciding whether or not to print their output.
Context managers: hide and show are twin context managers that take one or more output level names as strings, and either hide or show them within the wrapped block. As with Fabric’s other context managers, the prior values are restored when the block exits.
Command-line arguments: You may use the --hide and/or --show arguments to fab options and arguments, which behave exactly like the context managers of the same names (but are, naturally, globally applied) and take comma-separated strings as input.