@Package HtmlPrag @Subtitle Pragmatic Parsing and Emitting of HTML using SXML and SHTML @HomePage http://www.neilvandyke.org/htmlprag/ @Author Neil W. Van Dyke @AuthorEmail neil@@neilvandyke.org @Version 0.13 @Date 2005-02-23
DL: disable tests for Bigloo
(cond-expand ((or bigloo chicken gambit) (define-macro (%htmlprag:testeez . x) #t) ) (else (define-syntax %htmlprag:testeez (syntax-rules () ((_ x ...) ;;(testeez x ...) (error "Tests disabled.") ))) ))
(define %htmlprag:a2c integer->char)
(define (%htmlprag:append! a b) (cond ((null? a) b) ((null?  The parser will introduce new contiguous whitespace ;; (e.g., when text tokens are concatenated, invalid end tags are removed, ;; whitespace is irrelevant between certain elements), but then the parser ;; only has to worry about the first and last character of each string. ;; Perhaps the text tokens should have both leading and trailing whitespace ;; stripped, and contain flags for whether or not leading and trailing ;; whitespace occurred. (letrec ((no-token '()) ;; TODO: Maybe make this an option. (verbatim-to-eof-elems '(plaintext)) ;; TODO: Implement proper parsing of `verbatim-pair-elems' elements. ;; Note that we must support invalid termination like this: (verbatim-pair-elems '(script server style xmp)) (ws-chars (list #\space (%htmlprag:a2c 9) (%htmlprag:a2c 10) (%htmlprag:a2c 11) (%htmlprag:a2c 12) (%htmlprag:a2c 13))) (output-string->string-or-false (lambda (os) (let ((s (%htmlprag:gosc os))) (if (string=? s "") #f s)))) (output-string->symbol-or-false (lambda (os) (let ((s (output-string->string-or-false os))) (if s (string->symbol s) #f)))) ) (lambda (in normalized?) ;; TODO: Make a tokenizer option that causes XML namespace qualifiers to ;; be ignored. (letrec ( ;; Port buffer with inexpensive unread of one character and slightly ;; more expensive pushback of second character to unread. The ;; procedures themselves do no consing. The tokenizer currently ;; needs two-symbol lookahead, due to ambiguous "/" while parsing ;; element and attribute names, which could be either empty-tag ;; syntax or XML qualified names. (c #f) (next-c #f) (c-consumed? #t) (read-c (lambda () (if c-consumed? (if next-c (begin (set! c next-c) (set! next-c #f)) (set! c (read-char in))) (set! c-consumed? #t)))) (unread-c (lambda () (if c-consumed? (set! c-consumed? #f) ;; TODO: Procedure name in error message really ;; isn't "make-html-tokenizer"... (%htmlprag:error "make-html-tokenizer" "already unread:" c)))) (push-c (lambda (new-c) (if c-consumed? (begin (set! c new-c) (set! c-consumed? #f)) (if next-c (%htmlprag:error "make-html-tokenizer" "pushback full:" c) (begin (set! next-c c) (set! c new-c) (set! c-consumed? #f)))))) ;; TODO: These procedures are a temporary convenience for ;; enumerating the pertinent character classes, with an eye towards ;; removing redundant tests of character class. These procedures ;; should be eliminated in a future version. (c-eof? (lambda () (eof-object? c))) (c-amp? (lambda () (eqv? c #\&))) (c-apos? (lambda () (eqv? c #\'))) (c-bang? (lambda () (eqv? c #\!))) (c-colon? (lambda () (eqv? c #\:))) (c-quot? (lambda () (eqv? c #\"))) (c-equals? (lambda () (eqv? c #\=))) (c-gt? (lambda () (eqv? c #\>))) (c-lt? (lambda () (eqv? c #\<))) (c-minus? (lambda () (eqv? c #\-))) (c-pound? (lambda () (eqv? c #\#))) (c-ques? (lambda () (eqv? c #\?))) (c-semi? (lambda () (eqv? c #\;))) (c-slash? (lambda () (eqv? c #\/))) (c-splat? (lambda () (eqv? c #\*))) (c-lf? (lambda () (eqv? c #\newline))) (c-angle? (lambda () (memv c '(#\< #\>)))) (c-ws? (lambda () (memv c ws-chars))) (c-alpha? (lambda () (char-alphabetic? c))) (c-digit? (lambda () (char-numeric? c))) (c-alphanum? (lambda () (or (c-alpha?) (c-digit?)))) (c-hexlet? (lambda () (memv c '(#\a #\b #\c #\d #\e #\f #\A #\B #\C #\D #\E #\F)))) (skip-ws (lambda () (read-c) (if (c-ws?) (skip-ws) (unread-c)))) (make-start-token (if normalized? (lambda (name ns attrs) (list name (cons '@ attrs))) (lambda (name ns attrs) (if (null? attrs) (list name) (list name (cons '@ attrs)))))) (make-empty-token (lambda (name ns attrs) (cons shtml-empty-symbol (make-start-token name ns attrs)))) (make-end-token (if normalized? (lambda (name ns attrs) (list shtml-end-symbol name (cons '@ attrs))) (lambda (name ns attrs) (if (null? attrs) (list shtml-end-symbol name) (list shtml-end-symbol name (cons '@ attrs)))))) (make-comment-token (lambda (str) (list shtml-comment-symbol str))) (make-decl-token (lambda (parts) (cons shtml-decl-symbol parts))) (scan-qname ;; TODO: Make sure we don't accept local names that have "*", since ;; this can break SXML tools. Have to validate this afterwards if ;; "verbatim-safe?". Also check for "@" and maybe "@@". Check ;; qname parsing code, especially for verbatim mode. This is ;; important! (lambda (verbatim-safe?) ;; Note: If we accept some invalid local names, we only need two ;; symbols of lookahead to determine the end of a qname. (letrec ((os #f) (ns '()) (vcolons 0) (good-os (lambda () (or os (begin (set! os (open-output-string)) os))))) (let loop () (read-c) (cond ((c-eof?) #f) ((or (c-ws?) (c-splat?)) (if verbatim-safe? (unread-c))) ((or (c-angle?) (c-equals?) (c-quot?) (c-apos?)) (unread-c)) ((c-colon?) (or (null? ns) (set! ns (cons ":" ns))) (if os (begin (set! ns (cons (%htmlprag:gosc os) ns)) (set! os #f))) (loop)) ((c-slash?) (read-c) (cond ((or (c-eof?) (c-ws?) (c-equals?) (c-apos?) (c-quot?) (c-angle?) (c-splat?)) (unread-c) (push-c #\/)) (else (write-char #\/ (good-os)) (write-char c os) (loop)))) (else (write-char c (good-os)) (loop)))) (let ((ns (if (null? ns) #f (apply string-append (%htmlprag:reverse!ok ns)))) (local (if os (%htmlprag:gosc os) #f))) (if verbatim-safe? ;; TODO: Make sure we don't have ambiguous ":" or drop ;; any characters! (cons ns local) ;; Note: We represent "xml:" and "xmlns:" syntax as ;; normal qnames, for lack of something better to do with ;; them when we don't support XML namespaces. ;; ;; TODO: Local names are currently forced to lowercase, ;; since HTML is usually case-insensitive. If XML ;; namespaces are used, we might wish to keep local names ;; case-sensitive. (if local (if ns (if (or (string=? ns "xml") (string=? ns "xmlns")) (string->symbol (string-append ns ":" local)) (cons ns (string->symbol (%htmlprag:down!ok local)))) (string->symbol (%htmlprag:down!ok local))) (if ns (string->symbol (%htmlprag:down!ok ns)) ;; TODO: Ensure that it's OK to return #f as a ;; name. #f))))))) (scan-tag (lambda (start?) (skip-ws) (let ((tag-name (scan-qname #f)) (tag-ns #f) (tag-attrs #f) (tag-empty? #f)) ;; Scan element name. (if (pair? tag-name) (begin (set! tag-ns (car tag-name)) (set! tag-name (cdr tag-name)))) ;; TODO: Ensure there's no case in which a #f tag-name isn't ;; compensated for later. ;; ;; Scan element attributes. (set! tag-attrs (let scan-attr-list () (read-c) (cond ((c-eof?) '()) ((c-angle?) (unread-c) '()) ((c-slash?) (set! tag-empty? #t) (scan-attr-list)) ((c-alpha?) (unread-c) (let ((attr (scan-attr))) (cons attr (scan-attr-list)))) (else (scan-attr-list))))) ;; Find ">" or unnatural end. (let loop () (read-c) (cond ((c-eof?) no-token) ((c-slash?) (set! tag-empty? #t) (loop)) ((c-gt?) #f) ((c-ws?) (loop)) (else (unread-c)))) ;; Change the tokenizer mode if necessary. (cond ((not start?) #f) (tag-empty? #f) ;; TODO: Maybe make one alist lookup here, instead of ;; two. ((memq tag-name verbatim-to-eof-elems) (set! nexttok verbeof-nexttok)) ((memq tag-name verbatim-pair-elems) (set! nexttok (make-verbpair-nexttok tag-name)))) ;; Return a token object. (if start? (if tag-empty? (make-empty-token tag-name tag-ns tag-attrs) (make-start-token tag-name tag-ns tag-attrs)) (make-end-token tag-name tag-ns tag-attrs))))) (scan-attr (lambda () (let ((name (scan-qname #f)) (val #f)) (if (pair? name) (set! name (cdr name))) (let loop-equals-or-end () (read-c) (cond ((c-eof?) no-token) ((c-ws?) (loop-equals-or-end)) ((c-equals?) (let loop-quote-or-unquoted () (read-c) (cond ((c-eof?) no-token) ((c-ws?) (loop-quote-or-unquoted)) ((or (c-apos?) (c-quot?)) (let ((term c)) (set! val (open-output-string)) (let loop-quoted-val () (read-c) (cond ((c-eof?) #f) ((eqv? c term) #f) (else (write-char c val) (loop-quoted-val)))))) ((c-angle?) (unread-c)) (else (set! val (open-output-string)) (write-char c val) (let loop-unquoted-val () (read-c) (cond ((c-eof?) no-token) ((c-apos?) #f) ((c-quot?) #f) ((or (c-ws?) (c-angle?) ;;(c-slash?) ) (unread-c)) ;; Note: We can treat a slash in an ;; unquoted attribute value as a ;; value constituent because the ;; slash is specially-handled only ;; for XHTML, and XHTML attribute ;; values must always be quoted. We ;; could do lookahead for "/>", but ;; that wouldn't let us parse HTML ;; "<a href=/>" correctly, so this is ;; an easier and more correct way to ;; do things. (else (write-char c val) (loop-unquoted-val)))))))) (else (unread-c)))) (if normalized? (list name (if val (%htmlprag:gosc val) (symbol->string name))) (if val (list name (%htmlprag:gosc val)) (list name)))))) (scan-comment ;; TODO: Rewrite this to use tail recursion rather than a state ;; variable. (lambda () (let ((os (open-output-string)) (state 'start-minus)) (let loop () (read-c) (cond ((c-eof?) #f) ((c-minus?) (set! state (case state ((start-minus) 'start-minus-minus) ((start-minus-minus body) 'end-minus) ((end-minus) 'end-minus-minus) ((end-minus-minus) (write-char #\- os) state) (else (%htmlprag:error "make-html-tokenizer" "invalid state:" state)))) (loop)) ((and (c-gt?) (eq? state 'end-minus-minus)) #f) (else (case state ((end-minus) (write-char #\- os)) ((end-minus-minus) (display "--" os))) (set! state 'body) (write-char c os) (loop)))) (make-comment-token (%htmlprag:gosc os))))) (scan-pi (lambda () (skip-ws) (let ((name (open-output-string)) (val (open-output-string))) (let scan-name () (read-c) (cond ((c-eof?) #f) ((c-ws?) #f) ((c-alpha?) (write-char c name) (scan-name)) (else (unread-c)))) ;; TODO: Do we really want to emit #f for PI name? (set! name (output-string->symbol-or-false name)) (let scan-val () (read-c) (cond ((c-eof?) #f) ;; ((c-amp?) (display (scan-entity) val) ;; (scan-val)) ((c-ques?) (read-c) (cond ((c-eof?) (write-char #\? val)) ((c-gt?) #f) (else (write-char #\? val) (unread-c) (scan-val)))) (else (write-char c val) (scan-val)))) (list shtml-pi-symbol name (%htmlprag:gosc val))))) (scan-decl ;; TODO: Find if SXML includes declaration forms, and if so, use ;; whatever format SXML wants. ;; ;; TODO: Rewrite to eliminate state variables. (letrec ((scan-parts (lambda () (let ((part (open-output-string)) (nonsymbol? #f) (state 'before) (last? #f)) (let loop () (read-c) (cond ((c-eof?) #f) ((c-ws?) (case state ((before) (loop)) ((quoted) (write-char c part) (loop)))) ((and (c-gt?) (not (eq? state 'quoted))) (set! last? #t)) ((and (c-lt?) (not (eq? state 'quoted))) (unread-c)) ((c-quot?) (case state ((before) (set! state 'quoted) (loop)) ((unquoted) (unread-c)) ((quoted) #f))) (else (if (eq? state 'before) (set! state 'unquoted)) (set! nonsymbol? (or nonsymbol? (not (c-alphanum?)))) (write-char c part) (loop)))) (set! part (%htmlprag:gosc part)) (if (string=? part "") '() (cons (if (or (eq? state 'quoted) nonsymbol?) part ;; TODO: Normalize case of things we make ;; into symbols here. (string->symbol part)) (if last? '() (scan-parts)))))))) (lambda () (make-decl-token (scan-parts))))) (scan-entity (lambda () (read-c) (cond ((c-eof?) "&") ((c-alpha?) ;; TODO: Do entity names have a maximum length? (let ((name (open-output-string))) (write-char c name) (let loop () (read-c) (cond ((c-eof?) #f) ((c-alpha?) (write-char c name) (loop)) ((c-semi?) #f) (else (unread-c)))) (set! name (%htmlprag:gosc name)) ;; TODO: Make the entity map an option. (let ((pair (assoc name '(("amp" . "&") ("apos" . "'") ("gt" . ">") ("lt" . "<") ("quot" . "\""))))) (if pair (cdr pair) (make-shtml-entity name))))) ((c-pound?) (let ((num (open-output-string)) (hex? #f)) (read-c) (cond ((c-eof?) #f) ((memv c '(#\x #\X)) (set! hex? #t) (read-c))) (let loop () (cond ((c-eof?) #f) ((c-semi?) #f) ((or (c-digit?) (and hex? (c-hexlet?))) (write-char c num) (read-c) (loop)) (else (unread-c)))) (set! num (%htmlprag:gosc num)) (if (string=? num "") "&#;" (let ((n (string->number num (if hex? 16 10)))) (if (and (<= 32 n 255) (not (= n 127))) (string (%htmlprag:a2c n)) (make-shtml-entity n)))))) (else (unread-c) "&")))) (normal-nexttok (lambda () (read-c) (cond ((c-eof?) no-token) ((c-lt?) (let loop () (read-c) (cond ((c-eof?) "<") ((c-ws?) (loop)) ((c-slash?) (scan-tag #f)) ((c-ques?) (scan-pi)) ((c-bang?) (let loop () (read-c) (cond ((c-eof?) no-token) ((c-ws?) (loop)) ((c-minus?) (scan-comment)) (else (unread-c) (scan-decl))))) ((c-alpha?) (unread-c) (scan-tag #t)) (else (unread-c) "<")))) ((c-gt?) ">") (else (let ((os (open-output-string))) (let loop () (cond ((c-eof?) #f) ((c-angle?) (unread-c)) ((c-amp?) (let ((entity (scan-entity))) (if (string? entity) (begin (display entity os) (read-c) (loop)) (let ((saved-nexttok nexttok)) (set! nexttok (lambda () (set! nexttok saved-nexttok) entity)))))) (else (write-char c os) (or (c-lf?) (begin (read-c) (loop)))))) (let ((text (%htmlprag:gosc os))) (if (equal? text "") (nexttok) text))))))) (verbeof-nexttok (lambda () (read-c) (if (c-eof?) no-token (let ((os (open-output-string))) (let loop () (or (c-eof?) (begin (write-char c os) (or (c-lf?) (begin (read-c) (loop)))))) (%htmlprag:gosc os))))) (make-verbpair-nexttok (lambda (elem-name) (lambda () (let ((os (open-output-string))) ;; Accumulate up to a newline-terminated line. (let loop () (read-c) (cond ((c-eof?) ;; Got EOF in verbatim context, so set the normal ;; nextok procedure, then fall out of loop. (set! nexttok normal-nexttok)) ((c-lt?) ;; Got "<" in verbatim context, so get next ;; character. (read-c) (cond ((c-eof?) ;; Got "<" then EOF, so set to the normal ;; nexttok procedure, add the "<" to the ;; verbatim string, and fall out of loop. (set! nexttok normal-nexttok) (write-char #\< os)) ((c-slash?) ;; Got "</", so... (read-c) (cond ((c-eof?) (display "</" os)) ((c-alpha?) ;; Got "</" followed by alpha, so unread ;; the alpha, scan qname, compare... (unread-c) (let* ((vqname (scan-qname #t)) (ns (car vqname)) (local (cdr vqname))) ;; Note: We ignore XML namespace ;; qualifier for purposes of comparison. ;; ;; Note: We're interning strings here for ;; comparison when in theory there could ;; be many such unique interned strings ;; in a valid HTML document, although in ;; practice this should not be a problem. (if (and local (eqv? (string->symbol (%htmlprag:down local)) elem-name)) ;; This is the terminator tag, so ;; scan to the end of it, set the ;; nexttok, and fall out of the loop. (begin (let scan-to-end () (read-c) (cond ((c-eof?) #f) ((c-gt?) #f) ((c-lt?) (unread-c)) ((c-alpha?) (unread-c) ;; Note: This is an ;; expensive way to skip ;; over an attribute, but ;; in practice more ;; verbatim end tags will ;; not have attributes. (scan-attr) (scan-to-end)) (else (scan-to-end)))) (set! nexttok (lambda () (set! nexttok normal-nexttok) (make-end-token elem-name #f '())))) ;; This isn't the terminator tag, so ;; add to the verbatim string the ;; "</" and the characters of what we ;; were scanning as a qname, and ;; recurse in the loop. (begin (display "</" os) (if ns (begin (display ns os) (display ":" os))) (if local (display local os)) (loop))))) (else ;; Got "</" and non-alpha, so unread new ;; character, add the "</" to verbatim ;; string, then loop. (unread-c) (display "</" os) (loop)))) (else ;; Got "<" and non-slash, so unread the new ;; character, write the "<" to the verbatim ;; string, then loop. (unread-c) (write-char #\< os) (loop)))) (else ;; Got non-"<" in verbatim context, so just add it ;; to the buffer, then, if it's not a linefeed, fall ;; out of the loop so that the token can be ;; returned. (write-char c os) (or (c-lf?) (loop))))) ;; Return the accumulated line string, if non-null, or call ;; nexttok. (or (output-string->string-or-false os) (nexttok)))))) (nexttok #f)) (set! nexttok normal-nexttok) (lambda () (nexttok))))))
(define (tokenize-html in normalized?) (let ((next-tok (make-html-tokenizer in normalized?))) (let loop ((tok (next-tok))) (if (null? tok) '() (cons tok (loop (next-tok)))))))
(define (shtml-token-kind token) (cond ((string? token) shtml-text-symbol) ((list? token) (let ((s (list-ref token 0))) (if (memq s `(,shtml-comment-symbol ,shtml-decl-symbol ,shtml-empty-symbol ,shtml-end-symbol ,shtml-entity-symbol ,shtml-pi-symbol)) s shtml-start-symbol))) (else (%htmlprag:error "shtml-token-kind" "unrecognized token kind:" token))))
(define %htmlprag:empty-elements '(& area base br frame hr img input isindex keygen link meta object param spacer wbr))
(define parse-html/tokenizer ;; TODO: Document the algorithm, then see if rewriting as idiomatic Scheme ;; can make it more clear. (letrec ((empty-elements ;; TODO: Maybe make this an option. This might also be an ;; acceptable way to parse old HTML that uses the `p' element as a ;; paragraph terminator. %htmlprag:empty-elements) (parent-constraints ;; TODO: Maybe make this an option. '((area . (map)) (body . (html)) (caption . (table)) (colgroup . (table)) (dd . (dl)) (dt . (dl)) (frame . (frameset)) (head . (html)) (isindex . (head)) (li . (dir menu ol ul)) (meta . (head)) (noframes . (frameset)) (option . (select)) (p . (body td th)) (param . (applet)) (tbody . (table)) (td . (tr)) (th . (tr)) (thead . (table)) (title . (head)) (tr . (table tbody thead)))) (start-tag-name (lambda (tag-token) (car tag-token))) (end-tag-name (lambda (tag-token) (list-ref tag-token 1)))) (lambda (tokenizer normalized?) ;; Example `begs' value: ;; ;; ( ((head ...) . ( (title ...) )) ;; ((html ...) . ( (head ...) (*COMMENT* ...) )) ;; (#f . ( (html ...) (*DECL* doctype ...) )) ) (let ((begs (list (cons #f '())))) (letrec ((add-to-current-beg (lambda (tok) (set-cdr! (car begs) (cons tok (cdr (car begs)))))) (finish-all-begs (lambda () (let ((toplist #f)) (map (lambda (beg) (set! toplist (finish-beg beg))) begs) toplist))) (finish-beg (lambda (beg) (let ((start-tok (car beg))) (if start-tok (%htmlprag:append! (car beg) (%htmlprag:reverse!ok (cdr beg))) (%htmlprag:reverse!ok (cdr beg)))))) (finish-begs-to (lambda (name lst) (let* ((top (car lst)) (starttag (car top))) (cond ((not starttag) #f) ((eqv? name (start-tag-name starttag)) (set! begs (cdr lst)) (finish-beg top) #t) (else (if (finish-begs-to name (cdr lst)) (begin (finish-beg top) #t) #f)))))) (finish-begs-upto (lambda (parents lst) (let* ((top (car lst)) (starttag (car top))) (cond ((not starttag) #f) ((memq (start-tag-name starttag) parents) (set! begs lst) #t) (else (if (finish-begs-upto parents (cdr lst)) (begin (finish-beg top) #t) #f))))))) (let loop () (let ((tok (tokenizer))) (if (null? tok) (finish-all-begs) (let ((kind (shtml-token-kind tok))) (cond ((memv kind `(,shtml-comment-symbol ,shtml-decl-symbol ,shtml-entity-symbol ,shtml-pi-symbol ,shtml-text-symbol)) (add-to-current-beg tok)) ((eqv? kind shtml-start-symbol) (let* ((name (start-tag-name tok)) (cell (assq name parent-constraints))) (and cell (finish-begs-upto (cdr cell) begs)) (add-to-current-beg tok) (or (memq name empty-elements) (set! begs (cons (cons tok '()) begs))))) ((eqv? kind shtml-empty-symbol) ;; Empty tag token, so just add it to current ;; beginning while stripping off leading `*EMPTY*' ;; symbol so that the token becomes normal SXML ;; element syntax. (add-to-current-beg (cdr tok))) ((eqv? kind shtml-end-symbol) (let ((name (end-tag-name tok))) (if name ;; Try to finish to a start tag matching this ;; end tag. If none, just drop the token, ;; though we used to add it to the current ;; beginning. (finish-begs-to name begs) ;; We have an anonymous end tag, so match it ;; with the most recent beginning. If no ;; beginning to match, then just drop the ;; token, though we used to add it to the ;; current beginning. (and (car (car begs)) (begin (finish-beg (car begs)) (set! begs (cdr begs))))))) (else (%htmlprag:error "parse-html/tokenizer" "unknown tag kind:" kind))) (loop))))))))))
(define (%htmlprag:parse-html input normalized? top?) (let ((parse (lambda () (parse-html/tokenizer (make-html-tokenizer (cond ((input-port? input) input) ((string? input) (open-input-string input)) (else (%htmlprag:error "%htmlprag:parse-html" "invalid input type:" input))) normalized?) normalized?)))) (if top? (cons shtml-top-symbol (parse)) (parse))))
(define (html->sxml-0nf input) (%htmlprag:parse-html input #f #t))
(define (html->sxml-1nf input) (%htmlprag:parse-html input #f #t))
(define (html->sxml-2nf input) (%htmlprag:parse-html input #t #t))
(define html->sxml html->sxml-0nf)
(define html->shtml html->sxml-0nf)
(define (%htmlprag:write-shtml-as-html/fixed shtml out foreign-filter) (letrec ((write-shtml-text (lambda (str out) (let ((len (string-length str))) (let loop ((i 0)) (if (< i len) (begin (display (let ((c (string-ref str i))) (case c ;; ((#\") """) ((#\&) "&") ((#\<) "<") ((#\>) ">") (else c))) out) (loop (+ 1 i)))))))) (write-dquote-ampified (lambda (str out) ;; TODO: If we emit """, we really should parse it, and HTML ;; 4.01 says we should, but anachronisms in HTML create the potential ;; for nasty mutilation of URI in attribute values. (let ((len (string-length str))) (let loop ((i 0)) (if (< i len) (begin (display (let ((c (string-ref str i))) (if (eqv? c #\") """ c)) out) (loop (+ 1 i)))))))) (do-thing (lambda (thing) (cond ((string? thing) (write-shtml-text thing out)) ((list? thing) (if (not (null? thing)) (do-list-thing thing))) (else (do-thing (foreign-filter thing #f)))))) (do-list-thing (lambda (thing) (let ((head (car thing))) (cond ((symbol? head) ;; Head is a symbol, so... (cond ((eq? head shtml-comment-symbol) ;; TODO: Make sure the comment text doesn't contain a ;; comment end sequence. (display "<!-- " out) (let ((text (car (cdr thing)))) (if (string? text) ;; TODO: Enforce whitespace safety without ;; padding unnecessarily. ;; ;; (let ((len (string-length text))) ;; (if (= len 0) ;; (display #\space out) ;; (begin (if (not (eqv? ;; (string-ref text 0) ;; #\space)) (display text out) (%htmlprag:error "write-shtml-as-html" "invalid SHTML comment text:" thing))) (or (null? (cdr (cdr thing))) (%htmlprag:error "write-shtml-as-html" "invalid SHTML comment body:" thing)) (display " -->" out)) ((eq? head shtml-decl-symbol) (let ((head (car (cdr thing)))) (display "<!" out) (display (symbol->string head) out) (for-each (lambda (n) (cond ((symbol? n) (display #\space out) (display (symbol->string n) out)) ((string? n) (display " \"" out) (write-dquote-ampified n out) (display #\" out)) (else (%htmlprag:error "write-shtml-as-html" "invalid SHTML decl:" thing)))) (cdr (cdr thing))) (display #\> out))) ((eq? head shtml-pi-symbol) (display "<?"    ;; TODO: Write test cases for foreign-filter of HTML writing. ;; TODO: Write test cases for attribute values that aren't simple strings.


Introduction¶
Density Peaks Advanced clustering¶
The DPApipeline package implements the Density Peaks Advanced (DPA) clustering algorithm as introduced in the paper Automatic topography of high-dimensional data sets by non-parametric Density Peak clustering. The package offers the following features:
- Intrinsic dimensionality estimation by means of the TWO-NN algorithm, published in the Estimating the intrinsic dimension of datasets by a minimal neighborhood information paper.
- Adaptive k-NN Density estimation by means of the PAk algorithm, published in the Computing the free energy without collective variables paper.
- Advanced version of the DP clustering algorithm, published in the Clustering by fast search and find of density peaks paper, which includes an automatic search of cluster centers and assessment of statistical significance of the clusters
The top-level directory layout:
cd DPApipeline
ls -l
.
|-- data/ # Input and output files.
|-- docs/ # Documentation files.
|-- notebooks/ # Python scripts in Jupyter notebooks.
|-- Pipeline/ # Source files.
|-- README.rst
|-- requirements.txt
|-- run_ipynb2py_versioning.sh
|-- config.sh
|-- setup.py
Source files¶
The source Python codes are stored inside the Pipeline
folder:
cd Pipeline
ls -l
.
|-- ...
|-- Pipeline/
| |-- __init__.py
| |-- DPA.py # Python module implementing the DPA
| | # clustering algorithm.
| |
| |-- _DPA.pyx # Cython extension of the DPA module.
| |
| |-- PAk.py # Python module implementing the PAk
| | # density estimator.
| |
| |-- _PAk.pyx # Cython extension of the PAk module.
| |
| |-- NRmaxL.f90 # Fortran extension for the Newton-Rapson algorithm.
| |
| |-- twoNN.py # Python module implementing the TWO-NN
| # algorithm for the ID calculation.
|
|-- ...
Documentation files¶
Full documentation about the Python codes developed and the how-to instructions is created in the docs
folder using Sphinx.
The documentation in HTML format can be found in docs/_build/html/index.html
.
The DPApipeline.pdf
is in the docs/_build/rinioh
folder.
Jupyter notebooks¶
Examples of how-to run the DPA
, `PAk
and `twoNN
modules are provided as Jupyter notebooks in the notebooks
folder. Additional useful user-cases are available in the same folder.
.
|-- ...
|-- notebooks/
| |-- DPA_analysis.ipynb # Guided example of how-to run the Pipeline package.
|
|
|-- ...
Getting started¶
The source code of DPApipeline is on github DPApipeline repository.
You need the git
command in order to be able to clone it, and we
suggest you to use Python virtual environment in order to create a
controlled environment in which you can install DPApipeline as
normal user avoiding conflicts with system files or Python libraries.
The following section documents the steps required to install DPApipeline on a Linux or Windows/Mac computer.
Debian/Ubuntu¶
Run the following commands to create and activate a Python virtual environment with python virtualenv:
apt-get install git python-dev virtualenv*
virtualenv -p python3 venvdpa
. venvdpa/bin/activate
Windows¶
A possible setup makes use of Anaconda. It has preinstalled and configured packages for data analysis and it is available on all major platforms. It uses conda as package manager, in addition to the standard pip.
A versioning control can be installed by downloading git.
Run the following commands to activate the conda virtual environment:
conda create -n venvdpa
conda activate venvdpa
to list the available environments you can type conda info --envs
, and to deactivate an active environment use source deactivate
.
Installation¶
Assuming you already have the Python virtual enviroment installed and activated on your machine, run the following commands to download the DPApipeline source code:
git clone<EMAIL_ADDRESS>Install DPApipeline with the following commands:
cd dpapipeline
. compile.sh
Note that it is possible to check which packages are installed with the pip freeze
command.
Quickstart¶
A use-case example is provided in the DPA_analysis.ipynb jupyter notebook.
Pipeline package¶
Pipeline.DPA module¶
-
class
Pipeline.DPA.
DensityPeakAdvanced
(Z=1, metric='euclidean', densities=None, err_densities=None, k_hat=None, nn_distances=None, nn_indices=None, affinity='precomputed', density_algo='PAk', k_max=1000, D_thr=23.92812698, dim=None, dim_algo='twoNN', blockAn=True, block_ratio=5, frac=1, n_jobs=None)[source]¶ Bases:
sklearn.base.BaseEstimator
,sklearn.base.DensityMixin
Class definition for the non-parametric Density Peak clustering.
The default pipeline makes use of the PAk density estimator and of the TWO-NN intristic dimension estimator. The densities and the corresponding errors can also be provided as precomputed arrays.
Parameters: Z (float, default = 1) – The number of standard deviations, which fixes the level of statistical confidence at which one decides to consider a cluster meaningful.
metric (string, or callable) – The distance metric to use. If metric is a string, it must be one of the options allowed by scipy.spatial.distance.pdist for its metric parameter, or a metric listed in
VALID_METRIC = [precomputed, euclidean]
. If metric isprecomputed
, X is assumed to be a distance matrix. Alternatively, if metric is a callable function, it is called on each pair of instances (rows) and the resulting value recorded. The callable should take two arrays from X as input and return a value indicating the distance between them. Default iseuclidean
.densities (array [n_samples], default = None) – The logarithm of the density at each point. If provided, the following parameters are ignored:
density_algo
,k_max
,D_thr
.err_densities (array [n_samples], default = None) – The uncertainty in the density estimation, obtained by computing the inverse of the Fisher information matrix.
k_hat (array [n_samples], default = None) – The optimal number of neighbors for which the condition of constant density holds.
nn_distances (array [n_samples, k_max+1]) – Distances to the k_max neighbors of each points.
nn_indices (array [n_samples, k_max+1]) – Indices of the k_max neighbors of each points.
affinity (string or callable, default ‘precomputed’) –
- How to construct the affinity matrix.
nearest_neighbors
: construct the affinity matrix by computing a graph of nearest neighbors.rbf
: construct the affinity matrix using a radial basis function (RBF) kernel.precomputed
: interpretX
as a precomputed affinity matrix.precomputed_nearest_neighbors
: interpretX
as a sparse graph of precomputed nearest neighbors, and constructs the affinity matrix by selecting then_neighbors
nearest neighbors.- one of the kernels supported by
pairwise_kernels()
.
density_algo (string, default = “PAk”) – Define the algorithm to use as density estimator. It mast be one of the options allowed by
VALID_DENSITY = [PAk, kNN]
.k_max (int, default=1000) – This parameter is considered if density_algo is
PAk
orkNN
, it is ignored otherwise. k_max set the maximum number of nearest-neighbors considered by the density estimator. Ifdensity_algo=PAk
, k_max is used by the algorithm in the search for the largest number of neighborsk_hat
for which the condition of constant density holds, within a given level of confidence. Ifdensity_algo=kNN
, k_max set the number of neighbors to be used by the standard k-Nearest Neighbor algorithm. If the number of points in the sample N is less than the default value, k_max will be set automatically to the valueN/2
.D_thr (float, default=23.92812698) – This parameter is considered if density_algo is
PAk
, it is ignored otherwise. Set the level of confidence in the PAk density estimator. The default value corresponds to a p-value of \(10^{-6}\) for a \(\chiˆ2\) distribution with one degree of freedom.dim (int, default = None) – Intrinsic dimensionality of the sample. If dim is provided, the following parameters are ignored:
dim_algo
,blockAn
,block_ratio
,frac
.dim_algo (string, or callable, default=”twoNN”) – Method for intrinsic dimensionality calculation. If dim_algo is
auto
, dim is assumed to be equal to n_samples. If dim_algo is a string, it must be one of the options allowed byVALID_DIM = [auto, twoNN]
.blockAn (bool, default=True) – This parameter is considered if dim_algo is
twoNN
, it is ignored otherwise. If blockAn is True the algorithm perform a block analysis that allows discriminating the relevant dimensions as a function of the block size. This allows to study the stability of the estimation with respect to changes in the neighborhood size, which is crucial for ID estimations when the data lie on a manifold perturbed by a high-dimensional noise.block_ratio (int, default=5) – This parameter is considered if dim_algo is
twoNN
, it is ignored otherwise. Set the minimum size of the blocks as n_samples/block_ratio. IfblockAn=False
,block_ratio
is ignored.frac (float, default=1) – This parameter is considered if dim_algo is
twoNN
, it is ignored otherwise. Define the fraction of points in the data set used for ID calculation. By default the full data set is used.n_jobs (int or None, optional (default=None)) – The number of jobs to use for the computation. This works by computing each of the n_init runs in parallel.
None
means 1 unless in ajoblib.parallel_backend
context.-1
means using all processors. See Glossary for more details.
Variables: - labels (array [Nclus]) – The clustering labels assigned to each point in the data set.
- halos (array [Nclus]) – The clustering labels assigned to each point in the data set. Points identified as halos have label equal to zero.
- topography (array [Nclus, Nclus]) – Let be Nclus the number of clusters, the topography consists in a Nclus × Nclus symmetric matrix, in which the diagonal entries are the heights of the peaks and the off-diagonal entries are the heights of the saddle points.
- distances (array [n_samples, k_max+1]) – Distances to the k_max neighbors of each points. The point itself is included in the array.
- indices (array [n_samples, k_max+1]) – Indices of the k_max neighbors of each points. The point itself is included in the array.
- k_hat (array [n_samples], default = None) – The optimal number of neighbors for which the condition of constant density holds.
- centers (array [Nclus]) – The clustering labels assigned to each point in the data set.
Example
References
- d’Errico, E. Facco, A. Laio and A. Rodriguez, Automatic topography of high-dimensional data sets by non-parametric Density Peak clustering (2018) https://arxiv.org/abs/1802.10549
-
fit
(X, y=None)[source]¶ Fit the DPA clustering on the data.
Parameters: - X (array [n_samples, n_samples] if metric == “precomputed”, or,) – [n_samples, n_features] otherwise
The input samples. Similarities / affinities between
instances if
affinity='precomputed'
. - y (Ignored) – Not used, present here for API consistency by convention.
Returns: self – Returns self.
Return type: - X (array [n_samples, n_samples] if metric == “precomputed”, or,) – [n_samples, n_features] otherwise
The input samples. Similarities / affinities between
instances if
-
Pipeline.DPA.
_DensityPeakAdvanced
(densities, err_densities, k_hat, distances, indices, Z)[source]¶ Main function implementing the Density Peak Advanced clustering algorithm:
- Automatic detection of cluster centers
- Point assignament to clusters in order of decreasing g
- Topography reconstruction: search of saddle points and cluster merging
Parameters: - densities (array [n_samples]) – The logarithm of the density at each point.
- err_densities (array [n_samples]) – The uncertainty in the density estimation, obtained by computing the inverse of the Fisher information matrix.
- k_hat (array [n_samples]) – The optimal number of neighbors for which the condition of constant density holds.
- distances (array [n_samples, k_max+1]) – Distances to the k_max neighbors of each points. The point itself is included in the array.
- indices (array [n_samples, k_max+1]) – Indices of the k_max neighbors of each points. The point itself is included in the array.
- Z (float, default = 1) – The number of standard deviations, which fixes the level of statistical confidence at which one decides to consider a cluster meaningful.
Variables: - labels (array [Nclus]) – The clustering labels assigned to each point in the data set.
- halos (array [Nclus]) – The clustering labels assigned to each point in the data set. Points identified as halos have
clustering lable equal to
-1
. - topography (array [Nclus, Nclus]) – Let be Nclus the number of clusters, the topography consists in a Nclus × Nclus symmetric matrix, in which the diagonal entries are the heights of the peaks and the off-diagonal entries are the heights of the saddle points.
- centers (array [Nclus]) – The list of points identified as the centers of the Nclus statistically significant clusters.
Pipeline.PAk module¶
-
class
Pipeline.PAk.
PointAdaptive_kNN
(k_max=1000, D_thr=23.92812698, metric='euclidean', dim_algo='auto', nn_distances=None, nn_indices=None, blockAn=True, block_ratio=20, frac=1, dim=None, n_jobs=None)[source]¶ Bases:
sklearn.base.BaseEstimator
,sklearn.base.DensityMixin
Class definition for the Pointwise Adaptive k-NN density estimator.
Parameters: - k_max (int, default=1000) – The maximum number of nearest-neighbors considered by the procedure that returns the
largest number of neighbors
k_hat
for which the condition of constant density holds, within a given level of confidence. If the number of points in the sample N is less than the default value, k_max will be set automatically to the valueN/2
. - D_thr (float, default=23.92812698) – Set the level of confidence. The default value corresponds to a p-value of \(10^{-6}\) for a \(\chiˆ2\) distribution with one degree of freedom.
- metric (string, or callable) – The distance metric to use.
If metric is a string, it must be one of the options allowed by
scipy.spatial.distance.pdist for its metric parameter, or a metric listed in
VALID_METRIC = [precomputed, euclidean]
. If metric isprecomputed
, X is assumed to be a distance matrix. Alternatively, if metric is a callable function, it is called on each pair of instances (rows) and the resulting value recorded. The callable should take two arrays from X as input and return a value indicating the distance between them. Default iseuclidean
. - dim_algo (string, or callable) – Method for intrinsic dimensionality calculation. If dim_algo is
auto
, dim is assumed to be equal to n_samples. If dim_algo is a string, it must be one of the options allowed byVALID_DIM = [auto, twoNN]
. - nn_distances (array [n_samples, k_max+1]) – Distances to the k_max neighbors of each points.
- nn_indices (array [n_samples, k_max+1]) – Indices of the k_max neighbors of each points.
- blockAn (bool, default=True) – This parameter is considered if dim_algo is
twoNN
, it is ignored otherwise. If blockAn is True the algorithm perform a block analysis that allows discriminating the relevant dimensions as a function of the block size. This allows to study the stability of the estimation with respect to changes in the neighborhood size, which is crucial for ID estimations when the data lie on a manifold perturbed by a high-dimensional noise. - block_ratio (int, default=20) – This parameter is considered if dim_algo is
twoNN
, it is ignored otherwise. Set the minimum size of the blocks as n_samples/block_ratio. IfblockAn=False
, block_ratio is ignored. - frac (float, default=1) – This parameter is considered if dim_algo is
twoNN
, it is ignored otherwise. Define the fraction of points in the data set used for ID calculation. By default the full data set is used. - dim (int) – Intrinsic dimensionality of the sample. If dim is provided, dim_algo is ignored.
- n_jobs (int or None, optional (default=None)) – The number of jobs to use for the computation. This works by computing
each of the n_init runs in parallel.
None
means 1 unless in ajoblib.parallel_backend
context.-1
means using all processors. See Glossary for more details.
Variables: - distances (array [n_samples, k_max+1]) – Distances to the k_max neighbors of each points.
- indices (array [n_samples, k_max+1]) – Indices of the k_max neighbors of each points.
- densities (array [n_samples]) – The logarithm of the density at each point.
- err_densities (array [n_samples]) – The uncertainty in the density estimation, obtained by computing the inverse of the Fisher information matrix.
- k_hat (array [n_samples]) – The optimal number of neighbors for which the condition of constant density holds.
- dc (array [n_sample]) – The radius of the optimal neighborhood for each point.
Examples
>>> from DPA import PAk >>> import numpy as np >>> X = np.array([[1, 1], [2, 1], [1, 0], ... [4, 7], [3, 5], [3, 6]]) >>> est = PAk.PointAdaptive_kNN().fit(X) >>> est.distances_ array([[0. , 1. , 1. ], [0. , 1. , 1.41421356], [0. , 1. , 1.41421356], [0. , 1.41421356, 2.23606798], [0. , 1. , 2.23606798], [0. , 1. , 1.41421356]]) >>> est.indices_ array([[0, 1, 2], [1, 0, 2], [2, 0, 1], [3, 5, 4], [4, 5, 3], [5, 4, 3]])
References
- Rodriguez, M. d’Errico, E. Facco and A. Laio, Computing the free energy without collective variables. J. chemical theory computation 14, 1206–1215 (2018).
-
fit
(X, y=None)[source]¶ Fit the PAk estimator on the data.
Parameters: - X (array [n_samples, n_samples] if metric ==
precomputed
, or,) – [n_samples, n_features] otherwise The input samples. - y (Ignored) – Not used, present here for API consistency by convention.
Returns: self – Returns self.
Return type: - X (array [n_samples, n_samples] if metric ==
- k_max (int, default=1000) – The maximum number of nearest-neighbors considered by the procedure that returns the
largest number of neighbors
-
Pipeline.PAk.
_PointAdaptive_kNN
(distances, indices, k_max=1000, D_thr=23.92812698, dim=None)[source]¶ Main function implementing the Pointwise Adaptive k-NN density estimator.
Parameters: - distances (array [n_samples, k_max+1]) – Distances to the k_max neighbors of each points. The point is included.
- indices (array [n_samples, k_max+1]) – Indices of the k_max neighbors of each points. The point is included.
- k_max (int, default=1000) – The maximum number of nearest-neighbors considered by the procedure that returns the
largest number of neighbors
k_hat
for which the condition of constant density holds, within a given level of confidence. If the number of points in the sample N is less than the default value, k_max will be set automatically to the valueN/2
. - D_thr (float, default=23.92812698) – Set the level of confidence. The default value corresponds to a p-value of \(10^{-6}\) for a \(\chiˆ2\) distribution with one degree of freedom.
- dim (int) – Intrinsic dimensionality of the sample.
Variables: - densities (array [n_samples]) – The logarithm of the density at each point.
- err_densities (array [n_samples]) – The uncertainty in the density estimation, obtained by computing the inverse of the Fisher information matrix.
- k_hat (array [n_samples]) – The optimal number of neighbors for which the condition of constant density holds.
- dc (array [n_sample]) – The radius of the optimal neighborhood for each point.
Pipeline.twoNN module¶
-
class
Pipeline.twoNN.
twoNearestNeighbors
(metric='euclidean', blockAn=True, block_ratio=20, frac=1, n_jobs=None)[source]¶ Bases:
sklearn.base.BaseEstimator
,sklearn.base.DensityMixin
Class definition for TWO-NN: Intrinsic dimension estimator by a minimal neighborhood information.
The TWO-NN estimator uses only the distances to the first two nearest neighbors of each point.
Parameters: - metric (string, or callable) – The distance metric to use.
If metric is a string, it must be one of the options allowed by
scipy.spatial.distance.pdist for its metric parameter, or a metric listed in
VALID_METRIC = [precomputed, euclidean]
. If metric isprecomputed
, X is assumed to be a distance matrix. Alternatively, if metric is a callable function, it is called on each pair of instances (rows) and the resulting value recorded. The callable should take two arrays from X as input and return a value indicating the distance between them. Default iseuclidean
. - blockAn (bool, default=True) – If blockAn is True the algorithm perform a block analysis that allows discriminating the relevant dimensions as a function of the block size. This allows to study the stability of the estimation with respect to changes in the neighborhood size, which is crucial for ID estimations when the data lie on a manifold perturbed by a high-dimensional noise.
- block_ratio (int, default=20) – Set the minimum size of the blocks as n_samples/block_ratio. If
blockAn=False
, block_ratio is ignored. - frac (float, default=1) – Define the fraction of points in the data set used for ID calculation. By default the full data set is used.
- n_jobs (int or None, optional (default=None)) – The number of jobs to use for the computation. This works by computing
each of the n_init runs in parallel.
None
means 1 unless in ajoblib.parallel_backend
context.-1
means using all processors. See Glossary for more details.
Variables: dim (int) – The intrinsic dimensionality
References
- Facco, M. d’Errico, A. Rodriguez and A. Laio, Estimating the intrinsic dimension of datasets by a minimal neighborhood information. Sci. reports 7, 12140 (2017)
-
fit
(X, y=None)[source]¶ A reference implementation of a fitting function.
Parameters: - X (array [n_samples, n_samples] if metric ==
precomputed
, or,) – [n_samples, n_features] otherwise The input samples. - y (Ignored) – Not used, present here for API consistency by convention.
Returns: self – Returns self.
Return type: - X (array [n_samples, n_samples] if metric ==
- metric (string, or callable) – The distance metric to use.
If metric is a string, it must be one of the options allowed by
scipy.spatial.distance.pdist for its metric parameter, or a metric listed in
-
Pipeline.twoNN.
_twoNearestNeighbors
(distances, blockAn=True, block_ratio=20, frac=1)[source]¶ Main function for the TWO-NN estimator.
Parameters: - distances (array [n_samples, k_max]) – Distances to the k_max neighbors of each points.
- blockAn (bool, default=True) – If blockAn is True the algorithm perform a block analysis that allows discriminating the relevant dimensions as a function of the block size. This allows to study the stability of the estimation with respect to changes in the neighborhood size, which is crucial for ID estimations when the data lie on a manifold perturbed by a high-dimensional noise.
- block_ratio (int, default=20) – Set the minimum size of the blocks as n_samples/block_ratio. If
blockAn=False
, block_ratio is ignored. - frac (float, default=1) – Define the fraction of points in the data set used for ID calculation. By default the full data set is used.


I have taken shrooms before once before this experience but now I am more ready than ever to try them again. My friend P and myself have considered taking mushrooms for quite some time and when a friend of mine got an 1/8th I bought them happily. I had done alot of research on mushrooms so I felt prepaired but I don't think P knew what he was getting into.
It was a normal Saturday and we hung out at his house for a little while until his parents left. When they did leave we went downstairs and made ourselves some peanut butter and shrooms sandwhiches. We munched down happily as we went back upstairs into his older brother J's room to trip. Since I'm a good 40 lbs. lighter than P I felt the effects sooner than he did. At first it was a slight tingling in my hands that pulsed through my spine. Thankfully we took some Dramamine before the shrooms so I didn't feel that sick. As the body trip became more powerful I felt my mind and body clinging to reality. I quickly laid down on the bed and let everything flow through myself.
The first noticable aspect of the trip was the fact that time was irrelevent. I would glance at the clock and it would say 3:00 and then it would feel like hours and I would glance back at the clock and it would say 3:01. J was playing 'Vice City' with P while all this was happening. I felt the desire to describe what was happening but I felt that talking was not necessary. So I stared quietly at the city while listening to Metallica on the radio. Suddenly I started having intense vivid open eyed visuals. I was really ready for it, at first I was just staring up at the white ceiling and brown ceiling fan when all of a sudden white lace seemed to by strung all over the ceiling. As I looked closer I could see the patterns were very intricate. They seemed to be blowing around in a gust of wind, but the window was closed so I moved my hand up to pull some down so I could study the patterns more. Then the fan morphed into a brilliant eagle. Wow these shrooms were good.
P. started to feel some of the effects now and the two of us stared at the smoke rising above some insence we were burning. J left for a smoke and the two of us followed him outside. I could see ripples where J's feet landed and got the feeling we were walking on water. There's this giant tree in their front yard and once we stepped outside it was like another world. The tree turned into a pot nugget, thank God I didn't try to smoke it. The sun was like waves all in all it was very cool stuff. Once we went back inside however it was all different.
I still felt pretty good but P went to the bathroom and while he was washing his hands he looked in the mirror. Now I have heard of people seeing themselves grow old and die in mirrors while tripping but what P saw just made him question his own existance. He then proceeded to leave me and J to go into his own room and type frantically on his computer. This is some of what he wrote:
The truth is in us...being happy
Life is a cycle of moods, Listen to me happy, sad, either or, I want them all.
Life is
Life is a cycle of moods, the computer tells us this!! READ ME!!
Once he tried to save and the computer malfuntioned he felt that he was basically in 'The Matrix'.  He then labeled his room 'the room of truth' and J's room 'the room of security'. He left his room and to tell us in J's room about what he found out but in doing this he locked himself out of his room. He then turned on me and J saying that we were trying to keep him for real truth and knowledge that he could only get while in the room of truth. J basically spent the rest of P's trip trying to calm him down but P would say in the middle of nowhere 'Brave New World'! Meanwhile I was in J's room trying not to listen to P for fear of myself bugging out. After finally P to take a nap after 4 hours of tripping all settled down.
I know I will try shrooms again, as for P he's to scared to try them again.
Experience Reports are the writings and opinions of the individual authors who submit them.
Some of the activities described are dangerous and/or illegal and none are recommended by Erowid.


25 Nov 2014
I had a time getting JsDOSBox going using the current version (I could have been doing it wrong, I dunno). I did manage to get it working. Here’s how I hacked it together:
Needed:
- JsDOSBox
- Ubuntu 14.04 with Apache (or another webserver)
Issues (As of 11/14):
- Version 3 didn’t seem to have everything I needed.
- I was able to get version 2 to work with one change.
mkdir ~/game
cd ~/game
wget http://downloads.sourceforge.net/project/jsdosbox/version2/jsdosbox.zip
unzip jsdosbox.zip
wget https://jsdosbox.appspot.com/jsdosbox/jsdosbox.nocache.js
mv jsdosbox.nocache.js jsdosbox2/jsdosbox2.nocache.js
nano index.html
change :
<!-- input id="autorun" type="hidden" value="imgmount c http://localhost:8080/war/paper.zip -t zip !c:!paper.exe" -->
to :
<input id="autorun" type="hidden" value="YOURZIPFILE.zip -t zip !c:!YOURCOMMAND.exe">
mv ~/game /var/www/html
(or wherever you want to serve it from)
Hit the your new webpage from your PC.
Enjoy!
Sample: http://atechdad.com/games/duke-nukem
24 Nov 2014
What is a BBS?
A BBS (bulletin board system) is basically terminal software that is lets people connect and share information. These systems were widely used prior to mass adoption of the internet as a means to share software, send messages, and play games. These BBSs were normally accessed via dial-up and as such usually catered to regional audiences.
What you’ll need (or what I used..)
- Ubuntu 14.04
- Docker
- Syncronet
Let’s get started:
1. Install Docker
From digitalocean:
sudo apt-get update
sudo aptitude install linux-image-extra-`uname -r`
sudo sh -c "wget -qO- https://get.docker.io/gpg | apt-key add -"
sudo sh -c "echo deb http://get.docker.io/ubuntu docker main\
> /etc/apt/sources.list.d/docker.list"
sudo aptitude update
sudo aptitude install lxc-docker
2.  Now, build your docker image. This will take a little while.
sudo docker build -t bbs_img .
4. Now, let’s run the image. This will build a container and run /bin/bash, giving us a shell inside of it.
sudo docker run -i -t bbs_img /bin/bash
Here you can edit your config via /sbbs/exec/scfg
When you are done you can commit the changes you made to your container back to your image with:
sudo docker commit bbs bbs_img
Now, to run this as a single instance, use:
sudo docker run -i -t bbs_img /usr/bin/authbind /sbbs/exec/sbbs
This will start a fresh instance of this container. Usually with something like a BBS we’d want persistent storage. There are a couple of ways to do this. In this case I chose to run the container with a name– and then start that specific container each time:
sudo docker run -i -t --name bbs bbs_img /usr/bin/authbind /sbbs/exec/sbbs
next time I run this container, I’d use
sudo docker start bbs
It might also be a good idea to set up a cronjob to commit changes if this is the way you want to go.
Telnet to port 23.
Profit!
Disclaimer:
This is only a proof of concept. It’s not intended to be permanent — or even good. Use it to learn or as a starting point for something better. If you make any improvements, please post them in the comments below.
Further Reading:
Docker: https://docs.docker.com/userguide/
Synchronet: http://www.synchro.net/
BBS The Documentary: https://www.youtube.com/playlist?list=PLADB2C91B4BC96D20
The image above is a snapshot from https://www.youtube.com/user/SquirrelMonkeyCom
Why?
I did this to learn more about Docker and as an excuse to build a BBS.
23 Nov 2014
My son uses an educational website. One point of frustration for him– and us was that he would stray from the site and get lost in obscure places on the desktop. I put together this simple solution so when he logs in, he gets the website. It’s not perfect, but it works for us.
You’ll Need:
- Windows (This has been tested from XP to Windows 10 TechPreview. The details may not be exactly the same, but the method does work)
- Chrome Browser
- Login for the Kids
How it works:
Windows Operating systems load a shell, usually explorer.exe, when a user logs in. This hack replaces this shell with a custom Chrome. Chrome launches the website in fullscreen (kiosk) mode.
Let’s make it happen:
If you need instructions, please click on the link above.
2. Create a Login
From Microsoft:
-
To open User Accounts, click the
Start button
, click
Control Panel, click
User Accounts and Family Safety, and then click
User Accounts.
-
Click Manage another account. If you’re prompted for an administrator password or confirmation, type the password or provide confirmation.
-
Click Create a new account.
-
Type the name you want to give the user account, click an account type, and then click Create Account.
Since this account is meant for a 2 year old, I created the account with no password.
2. Log in, Create the Registry files
Log in as your target user. Create a text file named ReplaceShell.reg. Use this content– replace with your website of choice:
Windows Registry Editor Version 5.00
[HKEY_CURRENT_USER\Software\Microsoft\Windows NT\CurrentVersion\Winlogon]
"Shell"="\"C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe\" --kiosk http://atechdad.com"
Save the file and doubleclick on it. If all went well you should be prompted with a warning along the lines of, “Adding information can break your computer, etc. Do you want to continue?”. Click Yes. Log out.
Congratulations. If all went well, on your next login you should be met with the site you chose
3. HELP! How do I get out / get my shell back?
- Temporary: When logged in as the user without the default shell, press Ctrl-Alt-Delete. You should be greeted by either your task manager, or a window that allows you to launch your task manager. Launch it. Click File > New Task. Type explorer.exe and press okay. You should now have your familiar windows shell up.
- Permanent: Once you complete the above procedure, find your ReplaceShell.reg. Replace the text with this:
Windows Registry Editor Version 5.00
[HKEY_CURRENT_USER\Software\Microsoft\Windows NT\CurrentVersion\Winlogon]
"Shell"="explorer.exe"
Double-Click on the .reg file. Accept just as before and log out. You should be back to normal.
22 Nov 2014
Welcome to aTechDad.com!
I created this site to aggregate and archive all of my many, ever-changing interests and practice my writing. If anything posted here helps you, awesome!


Naomi Campbell reveals her 'heart is happy', months after Liam Payne 'split'
She has also come to realise she has to 'love herself'
by Robert LeighNaomi Campbell has declared her 'heart is happy' after falling in love with a mystery person.
The 49-year-old supermodel - who dated Liam Payne earlier this year - revealed her dating status to The Guardian.
Asked whether she is in love, Naomi replied: "Yes. It's [bleep]ing hectic! It's actually not someone that you know, but my heart… my heart is happy."
The London-born fashion icon also noted how she has come to realise she has to 'love herself' rather than expect to be 'validated' by love from someone else.
Read more: Prince Andrew 'let side down' over friendship with paedophile
Naomi also revealed she is still close to all of her exes "except one".
And it seems that the end of a relationship does not mean the conclusion of her caring about a former partner.
"You come for someone that I love and care about, I'm defending them," she insisted.
She also explained how therapy has helped her to "stay in the light" - and praised the likes of Prince William and Prince Harry for helping decrease stigma around mental health.
 I don't have depression, but mental health is something that I care about a lot and it makes me happy to know that [the UK] is taking mental health seriously now," she said.
Back in April, Naomi reportedly called off her romance with One Direction star Liam because she was "bored" of him.
The pair were thought to have begun secretly dating at the end of 2018 and subsequently appeared at many of the same events, including Hong Kong’s Formula E lunch in Japan.
An anonymous source told a tabloid newspaper: "Naomi has taken the decision to let things fizzle out. She’s just not into Liam any more and has cast him aside. Quite simply she got a bit bored.
"He was always the more keen, slightly needy one and, as far as she was concerned, they were never officially an item anyway. It’s always been very casual."
Leave us a comment on our Facebook page @EntertainmentDailyFix and let us know what you think of this story.


‘Killed Like Flies’: CPF Condemns Murder Of Cape Police Officer
by SG EditorThe Bishop Lavis community policing forum in Cape Town has condemned the killing of a police officer in the area.
Lieutenant colonel Andre Kay was shot and killed while off duty on Thursday.
He was attached to the provincial firearm, liquor and second-hand goods control unit.
According to police, Kay got into his vehicle just before 7 am when a suspect fired shots at him. He died on the scene.
CPF chairperson Graham Lindhorst said the incident raised concerns in the community.
“It’s heartbreaking to see that people who are meant to protect you are being targeted. It shows you how great the issues are in our area. If they’re not safe, then what about the ordinary person?”
Lindhorst said the attack took place almost one kilometre from the police station and added that government needed to do more to protect police officers.
“Government must pull out their strong arm and deal with these guys. We cannot have a situation where officers are killed like flies.”


ICE CREAME
HOME
Services
Works
Aboust us
contact
Login
Register Here
Name
Email
Password
Re-Enter Password


This is my personal blog and any views expressed here are mine. Wish we can make something different. ---I can do all the things.
Sorry, we've misplaced that URL or it's pointing to something that doesn't exist. Head back home to try finding it again.


General information about the pyboard¶
Contents
Local filesystem and SD card¶
There is a small internal filesystem (a drive) on the pyboard, called /flash
,
which is stored within the microcontroller’s flash memory. If a micro SD card
is inserted into the slot, it is available as /sd
.
When the pyboard boots up, it needs to choose a filesystem to boot from. If
there is no SD card, then it uses the internal filesystem /flash
as the boot
filesystem, otherwise, it uses the SD card /sd
. After the boot, the current
directory is set to one of the directories above.
If needed, you can prevent the use of the SD card by creating an empty file
called /flash/SKIPSD
. If this file exists when the pyboard boots
up then the SD card will be skipped and the pyboard will always boot from the
internal filesystem (in this case the SD card won’t be mounted but you can still
mount and use it later in your program using os.mount
).
(Note that on older versions of the board, /flash
is called 0:/
and /sd
is called 1:/
).
The boot filesystem is used for 2 things: it is the filesystem from which
the boot.py
and main.py
files are searched for, and it is the filesystem
which is made available on your PC over the USB cable.
The filesystem will be available as a USB flash drive on your PC. You can
save files to the drive, and edit boot.py
and main.py
.

Boot modes¶
If you power up normally, or press the reset button, the pyboard will boot
into standard mode: the boot.py
file will be executed first, then the
USB will be configured, then main.py
will run.
You can override this boot sequence by holding down the user switch as the board is booting up. Hold down user switch and press reset, and then as you continue to hold the user switch, the LEDs will count in binary. When the LEDs have reached the mode you want, let go of the user switch, the LEDs for the selected mode will flash quickly, and the board will boot.
The modes are:
Green LED only, standard boot: run
boot.py
thenmain.py
.Orange LED only, safe boot: don’t run any scripts on boot-up.
Green and orange LED together, filesystem reset: resets the flash filesystem to its factory state, then boots in safe mode.
If your filesystem becomes corrupt, boot into mode 3 to fix it. If resetting the filesystem while plugged into your compute doesn’t work, you can try doing the same procedure while the board is plugged into a USB charger, or other USB power supply without data connection.
Errors: flashing LEDs¶
There are currently 2 kinds of errors that you might see:
- If the red and green LEDs flash alternatively, then a Python script
(eg
main.py
) has an error. Use the REPL to debug it.
If all 4 LEDs cycle on and off slowly, then there was a hard fault. This cannot be recovered from and you need to do a hard reset.
Guide for using the pyboard with Windows¶
The following PDF guide gives information about using the pyboard with Windows, including setting up the serial prompt and downloading new firmware using DFU programming: PDF guide.
The pyboard hardware¶
For the pyboard:
- v1.1
PYBv1.1 schematics and layout (2.9MiB PDF)
- v1.0
PYBv1.0 schematics and layout (2.4MiB PDF)
PYBv1.0 metric dimensions (360KiB PDF)
PYBv1.0 imperial dimensions (360KiB PDF)
For the official skin modules:
LCD32MKv1.0 schematics (194KiB PDF)
AMPv1.0 schematics (209KiB PDF)
LCD160CRv1.0: see
lcd160cr
Datasheets for the components on the pyboard¶
The microcontroller: STM32F405RGT6 (link to manufacturer’s site)
The accelerometer: Freescale MMA7660 (800kiB PDF)
The LDO voltage regulator: Microchip MCP1802 (400kiB PDF)
Datasheets for other components¶
The LCD display on the LCD touch-sensor skin: Newhaven Display NHD-C12832A1Z-FSW-FBW-3V3 (460KiB PDF)
The touch sensor chip on the LCD touch-sensor skin: Freescale MPR121 (280KiB PDF)
The digital potentiometer on the audio skin: Microchip MCP4541 (2.7MiB PDF)


Toggle navigation
Karthik Balakrishna
Data Visualizations
Using D3
Plant-Based Food Alternatives
Care Guide for House Plants
Wine Selection by Taste
Cannabis Strains: Emotions and Medical Relief
Divergent Diet Theories: Paleo, Zone,and Vegan
Gerrymandering in District 12 of North Carolina
How the Trade War May Affect America's Employment and Economy
UnDACUmented: Who We Are
Government Oversight of Social Media Access + Privacy: // Government Data Requests, Content Restrictions, and Service Interruptions
Participation in the Recent Social Movements in the US
Global Water Stress: Current and Future
The Effects of Climate Change on the US Honey Bee Population
Urban Sprawl in the US and Increasing Air Pollution
Poverty and HDI in Africa
Suicides in Japan: Why? by Gender and Age
Hispanics in US: Education and Voting
Conflicts in Burundi
Ice Hockey Injuries
Interstate Highway System Growth: Reduced Fatalites and Commute Time


|
MindScape
1.0
MindScape é um jogo plataforma com elementos de terror e suspense. Como o jogo se situa na mente da protagonista, há monstros surreais que são criados com base em seus medos.
|
A audio class. More...
#include <audio.hpp>
Public Types | |
| enum | AudioType { MUSIC, CHUNK } |
Public Member Functions | |
| Audio (std::string p_name, std::string path, AudioType m_type, int repeat=0, int chanel=-1) | |
| bool | load () |
| This method load the audio. More... | |
| void | free () |
| Stop the audio of the game. More... | |
| void | play_effect () |
| This method play effects. More... | |
| void | play_music_type () |
| This method play music type. More... | |
| void | draw (int x, int y) |
| void | pause_music () |
| This method pause the music of the game. More... | |
| void | stop_effect () |
| This method stop the effect. More... | |
| void | set_repetitions (int repeat) |
| This method set the quantity of repetitions of the music. More... | |
| void | set_duration (float duration) |
| This method set the duration of the effect. More... | |
| void | set_effect_volume (int _volume) |
| This method set the volume of the effect. More... | |
| void | set_music_volume (int _volume) |
| This method set the volume of the music. More... | |
| Public Member Functions inherited from engine::Component | |
| Component (std::string p_name, std::pair< int, int > p_displacement, bool p_active, int p_priority) | |
| bool | is_active () |
| This Routine verifys if the component is active or deactive. More... | |
| void | activate () |
| This Routine activate the current component. More... | |
| void | deactivate () |
| This Routine deactivate the current component. More... | |
| std::pair< int, int > | get_displacement () |
| This Routine get the current component displacement. More... | |
| void | set_displacement (std::pair< int, int >) |
| This Routine set the current component displacement. More... | |
| int | get_priority () |
| This Routine get the current component priority. More... | |
| void | set_priority (int) |
| This Routine set the current component priority. More... | |
| std::string | get_name () |
| This Routine get the current component name. More... | |
| void | set_name (std::string) |
| This Routine set the current component name. More... | |
A audio class.
This class is a model that contains all specifications of the audio. The audio it is divided in music and chunk. That is used in different situations.
|
virtual |
Stop the audio of the game.
It is important to free audio, chunk and free memory, when program was closed.
Reimplemented from engine::Component.
|
virtual |
This method load the audio.
It is important to load and activate the audio, music or chunk, at the game. True to load is OK and false its not OK.
Reimplemented from engine::Component.
| void Audio::pause_music | ( | ) |
This method pause the music of the game.
It is important to pause a music when the player think it is necessary.
| void Audio::play_effect | ( | ) |
This method play effects.
It is important to play a effect based in the time and in the scene wheres player is.
| void Audio::play_music_type | ( | ) |
This method play music type.
It is important to play music based in the theme and to ambientize the player at the game. That is a validation to know if resume or play the music.
| void Audio::set_duration | ( | float | duration | ) |
This method set the duration of the effect.
It is important for the effect of the audio to last longer.
| integer | to know how the duration of the effect. |
| void Audio::set_effect_volume | ( | int | _volume | ) |
This method set the volume of the effect.
It is important to dont pass the limit of the volume. That is a condition to validate the limit.
| integer | to identify the nivel of the effect volume. |
| void Audio::set_music_volume | ( | int | _volume | ) |
This method set the volume of the music.
It is important to dont pass the limit of the volume. That is a validate to increase and to decrease volume of the music.
| integer | to identify the nivel of the music volume. |
| void Audio::set_repetitions | ( | int | repeat | ) |
This method set the quantity of repetitions of the music.
That is a condition to set the repeat.
| integer | to know how many times is to repeat the music. |
| void Audio::stop_effect | ( | ) |
This method stop the effect.
It is important to aplly this method to stop the effect after finish the duration/action.


My name is James Zhong, although I'm generally known as Thebitspud in online communities. I am a programmer who primarily works with Java, HTML, CSS (usually Bootstrap 4), JavaScript (mostly jQuery or node.js), and Python. Furthermore, I am currently learning C++ as well as the LibGDX and Phaser 3 game development frameworks.
Outside of coding, my hobbies include weight training, reading, gardening, and playing video games.
When it comes to programming, my two main focuses are web development and game development, although recently I have been prioritizing the former. Currently, I am doing freelance website development part-time to build up my portfolio.
If you're interested in hiring or working with me, feel free to send me an email.


Loading...
threefry2x32
threefry.h
threefry2x32_R
threefry.h
threefry2x32keyinit
threefry.h
threefry2x64
threefry.h
threefry2x64_R
threefry.h
threefry2x64keyinit
threefry.h
threefry4x32
threefry.h
threefry4x32_R
threefry.h
threefry4x32keyinit
threefry.h
threefry4x64
threefry.h
threefry4x64_R
threefry.h
threefry4x64keyinit
threefry.h
Searching...
No Matches


The main aim of this project is to study both classification and regression problems, starting with the regression algorithms studied in project 1. We will include logistic regresion for classification problems and write our own multilayer perceptron code for studying both regression and classification problems. The codes developed in project 1, including bootstrap and/or cross-validation as well as the computation of the mean-squared error and the R2 score function can also be utilized (and included in logistic regression and the neural network codes) in the present analysis.
We will use the so-called Ising model for our training data and will focus on supervised training. We will follow closely the recent article of Mehta et al, arXiv 1803.08823. This article stands out as an excellent review on machine learning (ML) algorithms. The added benefit is that each figure and model presented in this article is accompanied by its jupyter notebook. This means that we can start using these and compare with our own results. They provide also the data set for the regression and classification analysis that we will explore. In this sense, with their available notebooks, it makes life easier since we can compare our own codes with their codes.
With the abovementioned configurations we will determine, using first various regression methods, the value of the coupling constant for the energy of the one-dimensional Ising model. Thereafter, we will use the two-dimensional data, but now computed at different temperatures, in order to classify the phase of the Ising model. Below the critical temperature, the system will be in a so-called ferromagnetic phase. Close to the critical temperature, the final magnetization becomes smaller and smaller in absolute value while above the critical temperature, the net magnetization is zero. This classification case, that is the two-dimensional Ising model, will be studied using logistic regression and deep neural networks. The aim is to develop your own logistic regression code for the classification of the phases (this is a binary model) and your multilayer perceptron code for the classification and regression case. You can compare your own results with those obtained using scikit-learn or tensorflow or other Python packages such as keras or other.
Feel free to use the notebooks to benchmark your code. If you wish to write your own C++ or Fortran program for say a multilayer neural network model and a logistic regression model, please feel free to do so.
The model we will employ in our studies is the so-called Ising model. Together with models like the Potts model and similar so-called lattice models, the Ising model has been widely studied in mathematics (in statistics in particular), physics, life science, chemistry and even in the social sciences in order to model social behavior. It is a simple binary value system where the variables of the model (spins often in physics) can take two values only, for example \( \pm 1 \) or \( 0 \) and \( 1 \). The system exhibits a phase transition in two or higher dimensions and the first person to find the analytical expressions for various expectation values was the Norwegian chemist Lars Onsager (Nobel prize in chemistry) after a tour de force mathematics exercise.
In our discussions here we will stay with a physicist's approach and call the variables for spin. You could replace this with any other type of binary variables, ranging from a two political parties to blue and red spheres. In its simplest form we define the energy of the system as $$ \begin{equation*} E=-J\sum_{< kl>}^{N}s_ks_l, \end{equation*} $$ with \( s_k=\pm 1 \), \( N \) is the total number of spins, \( J \) is a coupling constant expressing the strength of the interaction between neighboring spins.
The symbol \( < kl> \) indicates that we sum over nearest neighbors only. Notice that for \( J>0 \) it is energetically favorable for neighboring spins to be aligned. This feature leads to, at low enough temperatures, a cooperative phenomenon called spontaneous magnetization. That is, through interactions between nearest neighbors, a given magnetic moment can influence the alignment of spins that are separated from the given spin by a macroscopic distance. These long range correlations between spins are associated with a long-range order in which the lattice has a net magnetization in the absence of a magnetic field.
We start by considering the one-dimensional Ising model with nearest neighbor interactions. This model does not exhibit any phase transition.
Consider the 1D Ising model with nearest-neighbor interactions $$ \begin{equation*} E[\hat{s}]=-J\sum_{j=1}^{N}s_{j}s_{j+1}, \end{equation*} $$
on a chain of length \( N \) with so-called periodic boundary conditions and \( S_j=\pm 1 \) Ising spin variables. In one dimension, this model has no phase transition at finite temperature.
In the Python code below we generate, with a coupling coefficient set to \( J=1 \), a large number of spin configurations say \( 10000 \) as shown in the code below. It means that our data will be a set of \( i=1\ldots n \) points of the form \( \{(E[\boldsymbol{s}^i],\boldsymbol{s}^i)\} \). Our task is to find the value of \( J \) from the data set using linear regression.
Here is the Python code you need to generate the training data, see also the notebook of Mehta et al.
import numpy as np
import scipy.sparse as sp
np.random.seed(12)
import warnings
#Comment this to turn on warnings
warnings.filterwarnings('ignore')
### define Ising model aprams
# system size
L=40
# create 10000 random Ising states
states=np.random.choice([-1, 1], size=(10000,L))
def ising_energies(states,L):
"""
This function calculates the energies of the states in the nn Ising Hamiltonian
"""
J=np.zeros((L,L),)
for i in range(L):
J[i,(i+1)%L]-=1.0
# compute energies
E = np.einsum('...i,ij,...j->...',states,J,states)
return E
# calculate Ising energies
energies=ising_energies(states,L)
We can now recast the problem as a linear regression model using our codes from project 1. The way we are going to build our model mimicks the way we could think of finding say the gravitional constant for the graviational force between two planets. In the absence of any prior knowledge, one sensible choice is the all-to-all Ising model $$ E_\mathrm{model}[\boldsymbol{s}^i] = - \sum_{j=1}^N \sum_{k=1}^N J_{j,k}s_{j}^is_{k}^i. $$
Here \( i \) represents a particular spin configuration (one of the possible \( n \) configurations we generated with the code above).
This model is uniquely defined by the non-local coupling strengths \( J_{jk} \) which we want to learn. The model is linear in \( \mathbf{J} \) which makes it possible to use linear regression.
To apply linear regression, we recast this model in the form $$ E_\mathrm{model}^i \equiv \mathbf{X}^i \cdot \mathbf{J}, $$
where the vectors \( \mathbf{X}^i \) represent all two-body interactions \( \{s_{j}^is_{k}^i \}_{j,k=1}^N \), and the index \( i \) runs over the samples in the data set. To make the analogy complete, we can also represent the dot product by a single index \( p = \{j,k\} \), i.e. \( \mathbf{X}^i \cdot \mathbf{J}=X^i_pJ_p \). Note that the regression model does not include the minus sign, so we expect to learn negative \( J \)'s.
With these preliminaries, we are now ready to reutilize our codes from project 1.
We start with the one-dimensional Ising model and use the data we have generated with \( J=1 \) in the previous point. Use linear regression, Lasso and Ridge regression as done in project 1. You can compare your results with those of Mehta et al.. Make sure it is the 1D data which is used.
Discuss the methods and how they perform in computing the coupling constant \( J \) and include a bias-variance analysis using either cross-validation or bootstrap. Discuss also the mean squared error and the \( R2 \) score as measures to assess your model.
Give a critical analysis of your results.
We switch now to binary classification methods and use logistic regression to define the phases of the Ising model. This means that we switch to the two-dimensional Ising model and use the data sets generated by Mehta et al These energies and their corresponding spin orientation configurations represent then your data. We will use a fixed lattice of \( L\times L = 40 \times 40 \) spins in two dimensions. The link above contains data for several temperatures. The theoretical critical temperature for a phase transition is \( T_C\approx 2.269 \) in units of energy. However, for a finite lattice the results representing the critical temperature are slightly higher (\( T_C \approx 2.3 \)).
Our goal here, using logistic regression, is to train our model to predict the phase of a sample given the spin configuration, whether it represents a state above the critical temperature or below. The configurations representing states below the critical temperature are called ordered states (the spins tend to point in one direction, resulting in a net magnetic moment) while those above the critical temperature are called disordered. Since a finite lattice like this does not exhibit a clear sign of a phase transition we will mainly stay with either orderer or disordered phases. You could include the critical phase if you want.
Your aim here is thus to read in these data (use the examples from Mehta et al) and write your own code for doing logistic regression, see the lecture notes on logistic regression.
In this case, to evaluate the model, we will use the so-called accuracy score instead of the bootstrap or cross-validation as done in the standard linear regression part discussed in b). Examples of how to define the accuracy score can be found under the neural network slides, see for example the slides here.
To measure the performance of our network we evaluate how well it does it data it has never seen before, i.e. the test data. We measure the performance of the network using the accuracy score. The accuracy is as you would expect just the number of images correctly labeled divided by the total number of images. A perfect classifier will have an accuracy score of \( 1 \). $$ \text{Accuracy} = \frac{\sum_{i=1}^n I(t_i = y_i)}{n} , $$
where \( I \) is the indicator function, \( 1 \) if \( t_i = y_i \) and \( 0 \) otherwise, where \( t_i \) represents the target and \( y_i \) the outputs.
In order to find the optimal parameters of your logistic regressor you should include a gradient descent solver, as discussed in the gradient descent lectures. Since we don't have so many data points, you may just code the standard gradient descent with a given learning rate, or even attempt to use the Newton-Raphson method. Alternatively, it may be useful for the next part on neural networks to implement a stochastic gradient descent with and without mini-batches. Stochastic gradient with mini-batches may give the best results. You could finally compare your code with the output from scikit-learn's toolbox for optimization methods applied to logistic regression.
The notebook of Mehta et al is highly recommended in order to benchmark your code and results.
Your aim now, and this is the central part of this project, is to write to your own multilayer perceptron model implementing the back propagation algorithm discussed in the lecture slides. We start with the regression case discussed in parts a) and b) but train now the network to find the optimal weights and biases. You are free to use the codes in the above lecture slides as starting points.
Train your network and compare the results with those from your linear regression code. You can test your results against a similar code using _scikit_learn_ (see the examples in the above lecture notes) or tensorflow/keras.
A useful reference on the back progagation algorithm is Nielsen's book. It is an excellent read.
Finally, change now your cost function to the \( log \) cross-entropy classification cost function for the case discussed in part c). Train your network again and compare the results with those from your logistic regression code i c). Here again you can compare your results with those of Mehta et al. There they used tensorflow to classify the phases.
After all these glorious calculations, you should now summarize the various algorithms and come with a critical evaluation of their pros and cons. Which algorithm works best for the regression case and which is best for the classification case. These codes will also be part of your final project 3, but now applied to other data sets.
Here follows a brief recipe and recommendation on how to write a report for each project.
The preferred format for the report is a PDF file. You can also use DOC or postscript formats or as an ipython notebook file. As programming language we prefer that you choose between C/C++, Fortran2008 or Python. The following prescription should be followed when preparing the report:
If you have Python installed (we recommend Python3) and you feel pretty familiar with installing different packages, we recommend that you install the following Python packages via pip as
See below for a discussion of tensorflow and scikit-learn.
For OSX users we recommend also, after having installed Xcode, to install brew. Brew allows for a seamless installation of additional software via for example
If you don't want to install various Python packages with their dependencies separately, we recommend two widely used distrubutions which set up all relevant dependencies for Python, namely


Girish Godage, Experienced change management professional with successful track record of delivering high-profile, complex change programs in IT Application & Infrastructure. Vast practical knowledge of how to drive engagement and implement change that meets best practice goals and maximizes ROI. Areas of expertise and passions include strategy, business transformation, governance, process and organizational design, program, and project management.
I'm a passionate python coder. I like to code on my free time. I have created few repositories on github, which I made using Python language. One of them include a Tweeter Sentitment to CSV . I like to code in a short and sweet way.
I developed websites with Angular and WebAPI(.net C#,java,Node). This Website is made With HTML and CSS.
I am a Passionate Designing & thinking of Web and Mobile application too. I Love to use my skills and experience to design and devloped application for various Industry.
Many businesses spend thousands of dollars on websites and Mobile apps only to find out later that their goals aren’t being met and their business hasn’t grown as expected. I’ll help you identify the business goals behind your project and deliver a solution that not only looks great, but helps your business grow beyond what you invest. through my consultancy services.
Your Web app ,Mobile app or system is fundamental to the success of your business. Whether it’s an internal software tool that empowers your employees to do their job effectively or the platform your users engage with on a daily basis, your product can help your business grow toward success or decline into failure.A strategically designed product can save your employees hours of time each week by making tasks more efficient, help your platform grow by engaging more users, or simplify tasks that make the users work easier and more intuitive.
We provides professional Software Maintenance & Support for our Custom Software Solutions as well as for most software systems and applications developed by third parties. We have evolved a mature and well-defined application support and maintenance process that allows us to effectively and efficiently capture the application environment as well as the business activity.
Would you like to know more or just discuss something?
I have worked on dozens of projects so I have picked only the latest for you.


IDF Confirms: Mortar Shell Lands in Open Field
by Hana Levi JulianThe IDF has confirmed that a mortar shell landed Monday evening in an open field in the Eshkol Regional Council district after being fired at southern Israel from Gaza.
There are no specific details on exactly where the shell landed, although according to a spokesperson for the Eshkol district, security personnel are searching for the projectile.
The launch did trigger warnings on the smartphone application of the IDF Home Front Command, but the Red Alert incoming rocket alert siren did not activate, since the shell was not heading towards a populated area.


Install latest version of useful packages
brew install curl vim git
It's always good to override this basics


Book My Table
Home
Contact Us
Previous
Next
LIST OF CONTACTS
Party Name
Email
Phone
John Smith
<EMAIL_ADDRESS>999-999-9999


Transparency
Move this window to the projector and fullscreen
Click anywhere on this screen to open a new controller


World Toilet Day: The lives of Indian sanitation workers
Sudharak Olwe has been documenting the lives of Mumbai's sanitation workers for about two decades.
The work, often in appalling conditions, is reserved for Scheduled Castes, officially designated groups of historically disadvantaged communities that live on the fringes of society.
And their lives remain substantially unchanged despite India's overall economic, social and technological advancements.
Olwe's most recent photographs, commissioned by WaterAid, are shown as part of UN World Toilet Day 2019.
"Manual scavengers" from the Valmiki community remove excrement by hand from dry latrines in Amanganj, Panna, Madhya Pradesh.
Betibai Valmiki says: "We are not allowed to drink tea in any restaurant here.
"Even if we go to one small tea-shop, we are served in disposable plastic glasses while others are served in regular tumblers."
Most of the women have asthma and malaria - but there is no healthcare and their wages are docked if they call in sick.
Mukeshdevi, 42, pictured with her husband, Sukhraj, mother in-law, five children and two grandchildren, in Bhagwat Pura, Meerut, Uttar Pradesh, earns 2,000 rupees (£20) a month.
"What other option do we have?" she asks.
"Even if we open a shop, no-one would buy from us because we are Valmikis."
Santosh works in Amanganj with his wife and two sons.
In 1992, he nearly drowned cleaning a septic tank with colleagues, one of whom died.
It was much deeper than they had been told.
But despite his eyes being permanently damaged, he has never received any compensation.
Printed on the back of his jacket are the words "Being Human".
In Agra Mohalla, Panna Geeta Mattu, Sashi Balmeek and Raju Dumar work every day from 05:00 to 13:00 for 7,000 rupees a month.
"There is hardly any respect in it," Geeta says.
"We are treated so badly. It's such a thankless job."
In April last year, the Dom community on the outskirts of Thillai Gaon, Bihar, lost 10 houses and most of their cattle in a fire.
They work in nearby Sasaram but, having lost their ID and ration cards, received no help or compensation.
Meenadevi, 58, carries excrement from a Muslim neighbourhood in Rohtas.
She started working as a manual scavenger 25 years ago with her mother-in-law.
"Initially, I used to feel nauseated," she says.
"I wasn't ready and felt ashamed to work because of the stigma attached to it.
"But now I'm used to the foul smells.
"Poverty leaves you with no option.
"My mother-in-law died doing this job.
"She used to carry the sewage in tin cans. I did the same.
"Now, we don't use tin cans. Nonetheless, the same fate awaits me,"
All photographs copyright Sudharak Olwe and WaterAid


Many new pages are populated with a pluthera of images for buttons generated for sake of consistency. However even in this pattern of images many of the images do not match exactly in color - or even the effect of a click as a button.
Since we want our buttons to be uber-flexible, we'll have to make the background image expand with the size of the button's text. For that, we'll use the beloved Sliding Doors; two complementing images creating the illusion of a single, stretching image.
Our button will be a basic <a> tag with a nested <span> , each containing a different slice of the background image.


--- layout: default title: Parts of Sentences—A Trip Across Washington ---
Subject predicate When? Where? Why? How?
This sentence demonstrates how conjunctions such as and, but, and or are marked.
We can take an imaginary trip across our state. The journey starts in the morning. At the upper end of Grays Harbor, near the town of Aberdeen, we board a plane. As the plane ascends, you see another large bay to the south. It is called Willapa Bay. The Long Beach Peninsula is a long strip of land between Willapa Bay and the Pacific Ocean. Now we look north. We see thick evergreen forests right next to the shore. In the south there are sandy beaches, but in the north are many rocks.
The plane flies south of the Olympic Mountains. You can see snow on the tallest peaks all year long. The plane is now flying over gently rolling hills.
Soon you fly over a long, narrow body of water. George Vancouver explored this area by ship in the late 1700’s. He gratefully named it Puget Sound to honor an aide with him named Peter Puget. In the late 1700’s many Native Americans lived in the Puget Sound area. Now there are big cities along the shores of Puget Sound, and there are crowds of cars and trucks on its many highways.
Beyond the cities you can see farms among the forests. This part of the state is called the “Western Lowlands” because it is in the western part of the state and because it is near sea level. Several rivers flow through the Western Lowlands and into the Pacific Ocean.
Farther east, in the middle of the state, where the cities disappear, rise the Cascade Mountains. Snow covers many of these peaks all year long because they are so high. Like a tall wall, the Cascade Mountains keep most of the moisture on the west side of the state.
Because of the Cascade Mountains, much less rain falls on the east side of the state. The scenery changes suddenly on the east side of the mountains because it is so much drier. You can sometimes see green farmland among the dry grasslands, but farmers usually have to irrigate their land to make crops grow in this dry part of the state.
Much of the water for irrigation is taken from the Columbia River. This river flows from British Columbia south through eastern Washington and then east through a gap in the Cascades to the Pacific Ocean. The plane flies north now to the northeast corner of the state. We see the Columbia River. The river flows south through the Okanogan Highlands. Then it turns sharply west before it bends south and east around the Columbia Plateau. Around here, this part of the state is known as the Big Bend.
Now we see below us the foothills of the Rocky Mountains. Less rain falls here than in the Olympic Mountains or in the Cascade Mountains, so the forests grow less thickly. We have come to the Okanogan Highlands and nearly to the end of our journey. The plane will fly south now and take us to Spokane.


Purple MartiansTechnical Code DescriptionsNetgame - MainOverviewChangesOverview
Netgame is a method I use to synchronize a multiplayer Purple Martians game across different computers.
It is simple, robust, and works identically on linux and windows.
It uses a server-client model, with 1 server and up to 7 clients.
Like the rest of the Purple Martians project, it is based entirely on free software.
And the exact same source code is used to make both the windows and the linux versions.
I developed this method by trial and error. I explored and abandoned many different approaches.
All code is entirely my own, except for the low level libraries used:
libnet 0.10.2
A generic interface to a variety of network drivers.
Copyright (C) 1997-1999 Chad Catlett and George Foot
zlib
A Massively Spiffy Yet Delicately Unobtrusive Compression Library
(Also Free, Not to Mention Unencumbered by Patents)
I know there are many better implementations of network multiplayer out there. Other authors have done it much more thoroughly than I.
I did this because I wanted to. I wanted to see if I could do it on my own.
I love programming. Figuring out how synchronize running systems over networks was especially difficult and rewarding.
Changes
This is the latest iteration of netgame as of 2023
The biggest change in this latest version is that I have completely abandoned syncing game moves between server and client.
The only method used now is sending the server's game state to the clients.
The old version had client's inputs go through the server and return before they were applied.
The inescapable lag caused by this approach was one of the main reasons I abandoned it.
Now netgame will work over the internet, where before it would only work on a LAN with very low ping times.
In this version, clients run independently in between receiving new states from the server.
When client's controls change, they are applied to the client's local game state, saved locally on the client, and sent to the server.
The clients run slightly ahead of the server in time. That means that when a client receives a state from the server, it will most likely be in past.
Clients handle this by rewinding and replaying to the current frame. The rewind is simply applying a state from the past.
When the client replays, it re-applies its locally stored moves for frames beyond what the server has sent.
Because the server is running behind the clients, game moves from clients will most likely be received too late to be applied.
The server stores them in the the game moves array, even though they are in the past.
When the server sends states to clients, the server first rewinds and replays to apply the client's moves that arrived too late.
The end result is that the client's corrections when receiving server states are minimal.


Title:
Poor supply of petrol in petrol stations
Geolocation: Ikeja
Status:
Title:
Poor Hospital Equipments
Geolocation: Ikeja
Status:
Title:
Deface of lagos by pasting stickers and Poster
Geolocation: Ikeja
Status:
Title:
Poor learning condition in primary schools
Geolocation: Ikeja
Status:


EventPlanners has produced and managed many types of events and artists over the past decade. Read our promotions and events client reviews below. If you or your company has benefited from Sharon Batts and her special events team, please feel free to write a review and let us know about your experience. We pride ourselves on listening to you about your event ideas and making sure they come to life on your special day. Our client reviews say it all. See what our satisfied clients are saying about bWorks Entertainment. We would love to talk with you about your upcoming event. Get in touch. Click the Create Your Own Review below to write a review.
Feb 20, 2016 by Scott Smith, Field Sales Young & Company on EventPlanners
Sharon, on behalf of Young & Co and our line of manufacturer’s we represent, I wanted to thank you for letting us participate in the Safety-Kickoff event. Out of all of the ones I have been to in the last couple years this one was by far the best. The structure, organization, and PG&E employee participation was great!! Thank you and we look forward to participating in the next one.
Feb 20, 2016 by Tosha Lynette Stumon, Actress on bWorks Entertainment
I would like to say thank you to my talent manager; Sharon Batts. She tells me like it is and lets me know that every opportunity is not a good opportunity. She does not send me roles that are demeaning just to get paid. I respect her as a person and greatly appreciate her as my talent manager. Thank you for all you do for me.


About
Hello! My name's Novan and I'm a Data Scientist as well as a Javascript and Python developer. I've got a makers curiosity for development and a passion for the story that data tells. Outside of programming, you can find me doing Astro and Quantum Physics, Hip-Hop Dance, and music. Catch me dancing or singing my way around San Francisco!
Hello! My name's Novan and I'm a Data Scientist as well as a Javascript and Python developer. I've got a makers curiosity for development and a passion for the story that data tells. Outside of programming, you can find me doing Astro and Quantum Physics, Hip-Hop Dance, and music. Catch me dancing or singing my way around San Francisco!