| 1 | /* | |
| 2 | * Licensed to the Apache Software Foundation (ASF) under one or more | |
| 3 | * contributor license agreements. See the NOTICE file distributed with | |
| 4 | * this work for additional information regarding copyright ownership. | |
| 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 | |
| 6 | * (the "License"); you may not use this file except in compliance with | |
| 7 | * the License. You may obtain a copy of the License at | |
| 8 | * | |
| 9 | * http://www.apache.org/licenses/LICENSE-2.0 | |
| 10 | * | |
| 11 | * Unless required by applicable law or agreed to in writing, software | |
| 12 | * distributed under the License is distributed on an "AS IS" BASIS, | |
| 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| 14 | * See the License for the specific language governing permissions and | |
| 15 | * limitations under the License. | |
| 16 | */ | |
| 17 | package org.apache.commons.io.monitor; | |
| 18 | ||
| 19 | import java.util.List; | |
| 20 | import java.util.concurrent.CopyOnWriteArrayList; | |
| 21 | import java.util.concurrent.ThreadFactory; | |
| 22 | ||
| 23 | /** | |
| 24 | * A runnable that spawns a monitoring thread triggering any | |
| 25 | * registered {@link FileAlterationObserver} at a specified interval. | |
| 26 | * | |
| 27 | * @see FileAlterationObserver | |
| 28 | * @version $Id$ | |
| 29 | * @since 2.0 | |
| 30 | */ | |
| 31 | public final class FileAlterationMonitor implements Runnable { | |
| 32 | ||
| 33 | private final long interval; | |
| 34 | private final List<FileAlterationObserver> observers = new CopyOnWriteArrayList<>(); | |
| 35 | private Thread thread = null; | |
| 36 | private ThreadFactory threadFactory; | |
| 37 | private volatile boolean running = false; | |
| 38 | ||
| 39 | /** | |
| 40 | * Construct a monitor with a default interval of 10 seconds. | |
| 41 | */ | |
| 42 | public FileAlterationMonitor() { | |
| 43 | this(10000); | |
| 44 | } | |
| 45 | ||
| 46 | /** | |
| 47 | * Construct a monitor with the specified interval. | |
| 48 | * | |
| 49 | * @param interval The amount of time in milliseconds to wait between | |
| 50 | * checks of the file system | |
| 51 | */ | |
| 52 | public FileAlterationMonitor(final long interval) { | |
| 53 | this.interval = interval; | |
| 54 | } | |
| 55 | ||
| 56 | /** | |
| 57 | * Construct a monitor with the specified interval and set of observers. | |
| 58 | * | |
| 59 | * @param interval The amount of time in milliseconds to wait between | |
| 60 | * checks of the file system | |
| 61 | * @param observers The set of observers to add to the monitor. | |
| 62 | */ | |
| 63 | public FileAlterationMonitor(final long interval, final FileAlterationObserver... observers) { | |
| 64 | this(interval); | |
| 65 |
1
1. |
if (observers != null) { |
| 66 |
3
1. 2. 3. |
for (final FileAlterationObserver observer : observers) { |
| 67 |
1
1. |
addObserver(observer); |
| 68 | } | |
| 69 | } | |
| 70 | } | |
| 71 | ||
| 72 | /** | |
| 73 | * Return the interval. | |
| 74 | * | |
| 75 | * @return the interval | |
| 76 | */ | |
| 77 | public long getInterval() { | |
| 78 |
1
1. getInterval : replaced return of long value with value + 1 for org/apache/commons/io/monitor/FileAlterationMonitor::getInterval → KILLED |
return interval; |
| 79 | } | |
| 80 | ||
| 81 | /** | |
| 82 | * Set the thread factory. | |
| 83 | * | |
| 84 | * @param threadFactory the thread factory | |
| 85 | */ | |
| 86 | public synchronized void setThreadFactory(final ThreadFactory threadFactory) { | |
| 87 | this.threadFactory = threadFactory; | |
| 88 | } | |
| 89 | ||
| 90 | /** | |
| 91 | * Add a file system observer to this monitor. | |
| 92 | * | |
| 93 | * @param observer The file system observer to add | |
| 94 | */ | |
| 95 | public void addObserver(final FileAlterationObserver observer) { | |
| 96 |
1
1. addObserver : negated conditional → KILLED |
if (observer != null) { |
| 97 | observers.add(observer); | |
| 98 | } | |
| 99 | } | |
| 100 | ||
| 101 | /** | |
| 102 | * Remove a file system observer from this monitor. | |
| 103 | * | |
| 104 | * @param observer The file system observer to remove | |
| 105 | */ | |
| 106 | public void removeObserver(final FileAlterationObserver observer) { | |
| 107 |
1
1. removeObserver : negated conditional → KILLED |
if (observer != null) { |
| 108 |
1
1. removeObserver : negated conditional → SURVIVED |
while (observers.remove(observer)) { |
| 109 | } | |
| 110 | } | |
| 111 | } | |
| 112 | ||
| 113 | /** | |
| 114 | * Returns the set of {@link FileAlterationObserver} registered with | |
| 115 | * this monitor. | |
| 116 | * | |
| 117 | * @return The set of {@link FileAlterationObserver} | |
| 118 | */ | |
| 119 | public Iterable<FileAlterationObserver> getObservers() { | |
| 120 |
1
1. getObservers : mutated return of Object value for org/apache/commons/io/monitor/FileAlterationMonitor::getObservers to ( if (x != null) null else throw new RuntimeException ) → KILLED |
return observers; |
| 121 | } | |
| 122 | ||
| 123 | /** | |
| 124 | * Start monitoring. | |
| 125 | * | |
| 126 | * @throws Exception if an error occurs initializing the observer | |
| 127 | */ | |
| 128 | public synchronized void start() throws Exception { | |
| 129 |
1
1. start : negated conditional → KILLED |
if (running) { |
| 130 | throw new IllegalStateException("Monitor is already running"); | |
| 131 | } | |
| 132 | for (final FileAlterationObserver observer : observers) { | |
| 133 |
1
1. start : removed call to org/apache/commons/io/monitor/FileAlterationObserver::initialize → SURVIVED |
observer.initialize(); |
| 134 | } | |
| 135 | running = true; | |
| 136 |
1
1. start : negated conditional → KILLED |
if (threadFactory != null) { |
| 137 | thread = threadFactory.newThread(this); | |
| 138 | } else { | |
| 139 | thread = new Thread(this); | |
| 140 | } | |
| 141 |
1
1. start : removed call to java/lang/Thread::start → KILLED |
thread.start(); |
| 142 | } | |
| 143 | ||
| 144 | /** | |
| 145 | * Stop monitoring. | |
| 146 | * | |
| 147 | * @throws Exception if an error occurs initializing the observer | |
| 148 | */ | |
| 149 | public synchronized void stop() throws Exception { | |
| 150 |
1
1. stop : removed call to org/apache/commons/io/monitor/FileAlterationMonitor::stop → KILLED |
stop(interval); |
| 151 | } | |
| 152 | ||
| 153 | /** | |
| 154 | * Stop monitoring. | |
| 155 | * | |
| 156 | * @param stopInterval the amount of time in milliseconds to wait for the thread to finish. | |
| 157 | * A value of zero will wait until the thread is finished (see {@link Thread#join(long)}). | |
| 158 | * @throws Exception if an error occurs initializing the observer | |
| 159 | * @since 2.1 | |
| 160 | */ | |
| 161 | public synchronized void stop(final long stopInterval) throws Exception { | |
| 162 |
1
1. stop : negated conditional → KILLED |
if (running == false) { |
| 163 | throw new IllegalStateException("Monitor is not running"); | |
| 164 | } | |
| 165 | running = false; | |
| 166 | try { | |
| 167 |
1
1. stop : removed call to java/lang/Thread::join → SURVIVED |
thread.join(stopInterval); |
| 168 | } catch (final InterruptedException e) { | |
| 169 |
1
1. stop : removed call to java/lang/Thread::interrupt → NO_COVERAGE |
Thread.currentThread().interrupt(); |
| 170 | } | |
| 171 | for (final FileAlterationObserver observer : observers) { | |
| 172 |
1
1. stop : removed call to org/apache/commons/io/monitor/FileAlterationObserver::destroy → SURVIVED |
observer.destroy(); |
| 173 | } | |
| 174 | } | |
| 175 | ||
| 176 | /** | |
| 177 | * Run. | |
| 178 | */ | |
| 179 | @Override | |
| 180 | public void run() { | |
| 181 |
1
1. run : negated conditional → KILLED |
while (running) { |
| 182 | for (final FileAlterationObserver observer : observers) { | |
| 183 |
1
1. run : removed call to org/apache/commons/io/monitor/FileAlterationObserver::checkAndNotify → KILLED |
observer.checkAndNotify(); |
| 184 | } | |
| 185 |
1
1. run : negated conditional → KILLED |
if (!running) { |
| 186 | break; | |
| 187 | } | |
| 188 | try { | |
| 189 |
1
1. run : removed call to java/lang/Thread::sleep → SURVIVED |
Thread.sleep(interval); |
| 190 | } catch (final InterruptedException ignored) { | |
| 191 | } | |
| 192 | } | |
| 193 | } | |
| 194 | } | |
Mutations | ||
| 65 |
1.1 |
|
| 66 |
1.1 2.2 3.3 |
|
| 67 |
1.1 |
|
| 78 |
1.1 |
|
| 96 |
1.1 |
|
| 107 |
1.1 |
|
| 108 |
1.1 |
|
| 120 |
1.1 |
|
| 129 |
1.1 |
|
| 133 |
1.1 |
|
| 136 |
1.1 |
|
| 141 |
1.1 |
|
| 150 |
1.1 |
|
| 162 |
1.1 |
|
| 167 |
1.1 |
|
| 169 |
1.1 |
|
| 172 |
1.1 |
|
| 181 |
1.1 |
|
| 183 |
1.1 |
|
| 185 |
1.1 |
|
| 189 |
1.1 |


---
layout: post
title: Lonely Coffee Cherry
date: 2008-12-15 19:32:23.000000000 +05:30
categories:
- kiranoncemore
- S3 IS
tags:
- cherry
- coffee
status: publish
type: post
published: true
meta:
_edit_last: '2'
author:
login: kiranoncemore
email<EMAIL_ADDRESS>display_name: kiranoncemore
first_name: Kiran
last_name: ''
---
Tv: 1/400 s; Av: F/4.5; f: 6 mm (35mm-equiv: 38 mm)
Post-processing using Picasa (I'm feeling lucky)
This picture was shot at my native village where coffee cherries were being dried prior to extracting the coffee bean. Here are more details of coffee processing.
Some of the things I like about this photo:
- This snap is an example of how the rule-of-thirds could be used to create a visually pleasing effect. This snap might not have been as pleasing had the coffee cherry been smack in the center of the snap!
- I liked the shadow effect to some extent.
- The depth-of-field is just right. The subject itself, and the area immediately surrounding it; is in focus. There is that gradual loss of focus as you move away from the main subject.
Having said that, here are some flip-sides of this snap:
- Camera-shake - Even though I was shooting at 1/400s, and I was making good use of the "flip-and-rotate"-able LCD screen on my camera; I was still not able to fully eliminate the camera shake. My unbalanced posture, and the camera position (with the lens just inches from the ground) dint help either. However, I think I could have gotten the perfect still hand, had I tried a bit more.
- Shadow angle - yes; the same shadow effect which I liked is also a negative for this snap. If you notice, some part of the cherry is under its own shadow! I somehow feel if I had shot this thing from the other side, I would have got the shadow effect; minus the darkening of the subject itself.
- Size reference - It would have been nice if I had placed some other object unobtrusively in the picture (perhaps a flower), to give viewers an idea of the size of the subject. There are people who thought this thing was a dried coconut!
Readers' comments are welcome as always!


High Output Management
I just finished reading High Output Management, and it’s easily one of the best management books I’ve ever read. Lots of real, tangible advice on how to be a better people manager.
It’s written by Andy Grove, the former CEO of Intel - he was there as 3rd employee up to a publicly traded $20B+ company. During his years there, he wrote this book on what he learned. These are my notes.
Several concepts in the book, but really focused around 3 main points:
- Management is about output
- A manager’s output is the output of org under and the adjacent org
- The team only does well when peak performance is elicited from each member of the team
Output
The most important thing about management is that it’s all about output. You are building a machine - a black box in which you put something in, and something else comes out. To build and then optimize this, there are 3 important pieces — planning, production/indicators, and improving.
Planning
Planning is about achieving a consistent outcome in a stable and timely way. To do this, you always have to start with your limiting factor. That is the element of the plan that the rest of the plan must follow.
Example - If you’re trying to make a quality breakfast plate quickly (egg, toast, bacon), the limiting factor is what will take the longest to cook (the egg). So you work out how to boil the egg and then work backwards from there to get the rest of the plan in line.
A note on plans — the plan itself is often useless, but the act of planning is useful. Plans need to be made like that of a firefighter — you’re not sure what/when/where, but you need to have a framework in place so that everyone knows how to respond in the case that there is a fire. That’s the type of plan to build.
Production & Indicators
After planning, you move onto production. Production is process, assembly, and testing.
To monitor production and ensure success, you need to have the tools in place both to measure progress as well as outcome. Not only to evaluate how you did, but also to then figure out how to improve.
There are 2 types of indicators to think through —
- Leading
- Trend
Production is again, like a black box. You put something in and something comes out. Leading indicators are like cutting small holes into your black box at certain steps to peer inside. You can then evaluate how things are going during production and if things are on track.
Trend indicators measure output against time. These should be compared both individually and globally to measure your output over time.
Remember that indicators are measures of output, not output itself. It’s easy to fall in love and over optimize for indicators instead of focusing on output. Don’t confuse motion with progress. To combat optimizing for indicators and/or motion, make sure to pair each of these indicators with something else that may be affected so you can measure the result and weight how it ultimately affects output.
Improving
As you move through production and you begin to measure each of these with different indicators, you will begin to think through how to optimize / improve these steps. There are 2 ways to improve things:
- Do it faster
- Do it differently
To do it faster, write out the steps and figure out how to reduce them. Automation is a good way.
To do it differently, you have to fundamentally change how you do things, i.e. from a 100% audit process to an exception based audit. Either work, it depends on the context.
Additionally, as production scales, the question then becomes around which parts of the machine do you continue to localize and which ones become centralized. You have to balance local (responsiveness) and centralized (scale).
People
To achieve anything, the manager is relying on the team. Therefore the manager’s only job is to elicit peak performance from the team.
There are 2 ways - training and motivation. Training is straightforward and there’s no magic to it, it just takes time and must be performed by the manager. As for motivation, there are 2 types — achievement and competence. Achievement is awards based (trophies, money, etc) and competence (i.e. violin player who practices all day to improve his craft). You have to tailor your leadership to respective types of motivation.
Steps to make sure you’re helping the team perform —
- Making sure you’re continually adding value
- Determine the TRM for your report
- Set and communicate your modes of control
- Make sure you’re spending your time on high leverage activities
- Review indicators/output
- Collect info & make decisions with the team.
Adding Value
Much of this is about managing other people. But first when managing yourself, always keep in mind —
- Are you adding value or just passing it along in the chain?
- Are you plugged into what’s happening, both inside and outside the company?
- Are you trying new things / ideas?
Task Relevant Maturity
TRM is an assessment of how capable someone is at doing that particular task. Higher means they’re more experienced, lower means less so. Your management style should depend not by person, but by their TRM for that particular task (it can be different for the same person across different tasks).
If their TRM is low, you need to have a structured program in place - what, when, how, etc. If it’s high, setting high level objectives/goals and reviews will suffice. And there are several shades of gray in the middle.
Modes of control
There are 3 modes of control when it comes to dealing with your team —
- Free Market Forces
- Contractual Obligations
- Values
Free market is when there is no management needed and the market sets the rules. Buying something at the flea market is an example of free market forces. All you can do is abide by the what is set.
Contractual obligations are when the transaction cost rises and so for consistency, there is an agreement in place. Hiring is an example - whoever is hired will do X job, and in turn, they will get paid for it. To manage this, you need review and policing functions to ensure that terms of contract are being met and held.
And values are when the context changes faster than decisions can be made. And so values in turn encapsulate a framework to make decisions. The only way to enforce these is to live them as an example.
Determine what is which, and communicate this over. What are the values and how to embody those, how will someone be evaluated for contractual obligations, etc.
Managerial Leverage
Manager’s output = output of his org + output of neighboring org that they influence.
You can be doing any number of things at one time. But to make sure you are getting the most out of your team, you have to make sure you are sending your time on high leverage activities. A few steps —
- Figure out limiting step
- Batch similar tasks
- Take control of your calendar - plan out time to do things
- Say no earlier (more expensive to say no after has gone through some production)
- Allow slack - don’t have a single unexpected event derail your whole day
There’s also negative leverage — meddling when you don’t know what you’re doing. Don’t be one of these managers.
Collecting Info and Making Decisions
Ultimately, managers have 2 actions — collect info, and make decisions. Unfortunately, meetings are the best way to do this. Doesn’t mean they have to suck, there are good meetings. We have just been exposed to so many bad meetings that they now carry a bad name. But good meetings do exist.
To classify, there are 2 types of meetings —
- Process
- Mission
Process meetings are regular updates and should be done with a production mindset — batch stuff together, review indicators, focus on outcomes, make decisions, etc. 3 types of these —
- 1-1
- The agenda should be set from the report
- How you manage them should be based on their TRM
- Get status updates around key indicators
- How they’re feeling in general
- Listen to problems and things that will become problems
- Staff Meeting
- Manager should set the agenda far before so team can review
- The team should be actively discussing, manager just supervising (focus on indicators/outputs, guide to decisions, etc)
- Only discuss things that affect more than 2 people
- Operation Review
- Similar to staff meeting, but a full meeting across various departments
- Each team should be presenting, and the manager should be moderating
Mission oriented meetings are centered around a particular topic and solving a problem. In order for this to be effective, someone needs to own it — makes sure the right people are there, organize agenda ahead of time, keep people focused, and get the results out to everyone after.
The output of meetings is 6 things:
- What decision
- When
- Who will decide
- Who to consult
- Who to get approval from
- Who to inform
Overall, a great read on people management and highly recommended. Again, tried to keep it short and group common things together. Let me know any thoughts / things I’m missing in the comments.
You Might Also Like
- How Google Works
- Joining As Management
- Invest In People, Not Ideas
- Hot Teams
- How To Scale With Your Startup


|
SimpleITK
2.0.0.dev49
|
Calculates the B-Spline coefficients of an image. Spline order may be from 0 to 5. More...
#include <sitkBSplineDecompositionImageFilter.h>
Public Types | |
| typedef RealPixelIDTypeList | PixelIDTypeList |
| typedef BSplineDecompositionImageFilter | Self |
| Public Types inherited from itk::simple::ImageFilter< 1 > | |
| typedef ImageFilter | Self |
| Public Types inherited from itk::simple::ProcessObject | |
| typedef ProcessObject | Self |
Public Member Functions | |
| BSplineDecompositionImageFilter () | |
| Image | Execute (const Image &image1) |
| Image | Execute (const Image &image1, uint32_t splineOrder) |
| std::string | GetName () const |
| uint32_t | GetSplineOrder () const |
| std::vector< double > | GetSplinePoles () const |
| Self & | SetSplineOrder (uint32_t SplineOrder) |
| std::string | ToString () const |
| virtual | ~BSplineDecompositionImageFilter () |
| Public Member Functions inherited from itk::simple::ImageFilter< 1 > | |
| ImageFilter () | |
| virtual | ~ImageFilter ()=0 |
| Public Member Functions inherited from itk::simple::ProcessObject | |
| virtual void | Abort () |
| virtual int | AddCommand (itk::simple::EventEnum event, itk::simple::Command &cmd) |
| Add a Command Object to observer the event. More... | |
| virtual float | GetProgress () const |
| An Active Measurement of the progress of execution. More... | |
| virtual bool | HasCommand (itk::simple::EventEnum event) const |
| Query of this object has any registered commands for event. More... | |
| ProcessObject () | |
| virtual void | RemoveAllCommands () |
| Remove all registered commands. More... | |
| virtual | ~ProcessObject () |
| virtual void | DebugOn () |
| virtual void | DebugOff () |
| virtual bool | GetDebug () const |
| virtual void | SetDebug (bool debugFlag) |
| virtual void | SetNumberOfThreads (unsigned int n) |
| virtual unsigned int | GetNumberOfThreads () const |
Private Types | |
| typedef Image(Self::* | MemberFunctionType) (const Image &image1) |
Private Member Functions | |
| template<class TImageType > | |
| Image | ExecuteInternal (const Image &image1) |
Private Attributes | |
| nsstd::auto_ptr< detail::MemberFunctionFactory< MemberFunctionType > > | m_MemberFactory |
| uint32_t | m_SplineOrder |
| std::vector< double > | m_SplinePoles |
Friends | |
| struct | detail::MemberFunctionAddressor< MemberFunctionType > |
Additional Inherited Members | |
| Static Public Member Functions inherited from itk::simple::ProcessObject | |
| static bool | GetGlobalDefaultDebug () |
| static void | GlobalDefaultDebugOff () |
| static void | GlobalDefaultDebugOn () |
| static void | SetGlobalDefaultDebug (bool debugFlag) |
| static void | GlobalWarningDisplayOn () |
| static void | GlobalWarningDisplayOff () |
| static void | SetGlobalWarningDisplay (bool flag) |
| static bool | GetGlobalWarningDisplay () |
| static void | SetGlobalDefaultNumberOfThreads (unsigned int n) |
| static unsigned int | GetGlobalDefaultNumberOfThreads () |
| static double | GetGlobalDefaultCoordinateTolerance () |
| Access the global tolerance to determine congruent spaces. More... | |
| static void | SetGlobalDefaultCoordinateTolerance (double) |
| Access the global tolerance to determine congruent spaces. More... | |
| static double | GetGlobalDefaultDirectionTolerance () |
| Access the global tolerance to determine congruent spaces. More... | |
| static void | SetGlobalDefaultDirectionTolerance (double) |
| Access the global tolerance to determine congruent spaces. More... | |
| Protected Member Functions inherited from itk::simple::ProcessObject | |
| virtual unsigned long | AddITKObserver (const itk::EventObject &, itk::Command *) |
| virtual itk::ProcessObject * | GetActiveProcess () |
| virtual void | OnActiveProcessDelete () |
| virtual void | onCommandDelete (const itk::simple::Command *cmd) SITK_NOEXCEPT |
| virtual void | PreUpdate (itk::ProcessObject *p) |
| virtual void | RemoveITKObserver (EventCommand &e) |
| Protected Member Functions inherited from itk::simple::NonCopyable | |
| NonCopyable () | |
| Static Protected Member Functions inherited from itk::simple::ImageFilter< 1 > | |
| static void | FixNonZeroIndex (TImageType *img) |
| Static Protected Member Functions inherited from itk::simple::ProcessObject | |
| template<class TImageType > | |
| static TImageType::ConstPointer | CastImageToITK (const Image &img) |
| template<class TPixelType , unsigned int VImageDimension, unsigned int VLength, template< typename, unsigned int > class TVector> | |
| static Image | CastITKToImage (itk::Image< TVector< TPixelType, VLength >, VImageDimension > *img) |
| template<class TImageType > | |
| static Image | CastITKToImage (TImageType *img) |
| static const itk::EventObject & | GetITKEventObject (EventEnum e) |
| template<typename T > | |
| static std::ostream & | ToStringHelper (std::ostream &os, const T &v) |
| static std::ostream & | ToStringHelper (std::ostream &os, const char &v) |
| static std::ostream & | ToStringHelper (std::ostream &os, const signed char &v) |
| static std::ostream & | ToStringHelper (std::ostream &os, const unsigned char &v) |
Calculates the B-Spline coefficients of an image. Spline order may be from 0 to 5.
This class defines N-Dimension B-Spline transformation. It is based on: [1] M. Unser, "Splines: A Perfect Fit for Signal and Image Processing," IEEE Signal Processing Magazine, vol. 16, no. 6, pp. 22-38, November 1999. [2] M. Unser, A. Aldroubi and M. Eden, "B-Spline Signal Processing: Part I--Theory," IEEE Transactions on Signal Processing, vol. 41, no. 2, pp. 821-832, February 1993. [3] M. Unser, A. Aldroubi and M. Eden, "B-Spline Signal Processing: Part II--Efficient Design and Applications," IEEE Transactions on Signal Processing, vol. 41, no. 2, pp. 834-848, February 1993. And code obtained from bigwww.epfl.ch by Philippe Thevenaz
Limitations: Spline order must be between 0 and 5. Spline order must be set before setting the image. Uses mirror boundary conditions. Requires the same order of Spline for each dimension. Can only process LargestPossibleRegion
Definition at line 46 of file sitkBSplineDecompositionImageFilter.h.
|
private |
Setup for member function dispatching
Definition at line 96 of file sitkBSplineDecompositionImageFilter.h.
Define the pixels types supported by this filter
Definition at line 58 of file sitkBSplineDecompositionImageFilter.h.
Definition at line 48 of file sitkBSplineDecompositionImageFilter.h.
|
virtual |
Destructor
| itk::simple::BSplineDecompositionImageFilter::BSplineDecompositionImageFilter | ( | ) |
Default Constructor that takes no arguments and initializes default parameters
Execute the filter on the input image
| Image itk::simple::BSplineDecompositionImageFilter::Execute | ( | const Image & | image1, |
| uint32_t | splineOrder | ||
| ) |
Execute the filter on the input image with the given parameters
|
private |
|
inlinevirtual |
Name of this class
Implements itk::simple::ProcessObject.
Definition at line 79 of file sitkBSplineDecompositionImageFilter.h.
|
inline |
Definition at line 69 of file sitkBSplineDecompositionImageFilter.h.
|
inline |
Get the poles calculated for a given spline order.
This is a measurement. Its value is updated in the Execute methods, so the value will only be valid after an execution.
Definition at line 76 of file sitkBSplineDecompositionImageFilter.h.
Get/Sets the Spline Order, supports 0th - 5th order splines. The default is a 3rd order spline.
Definition at line 65 of file sitkBSplineDecompositionImageFilter.h.
|
virtual |
Print ourselves out
Reimplemented from itk::simple::ProcessObject.
|
friend |
Definition at line 100 of file sitkBSplineDecompositionImageFilter.h.
|
private |
Definition at line 102 of file sitkBSplineDecompositionImageFilter.h.
|
private |
Definition at line 106 of file sitkBSplineDecompositionImageFilter.h.
|
private |
Definition at line 108 of file sitkBSplineDecompositionImageFilter.h.


This paragraph is 70% opaque (or 30% transparent). Play with opacity value to see how it works.
opacity


{% extends "index-base.html" %} {% block header %} Create Knowledge {% endblock %} {% block content %}
| Description | Action |
|---|---|
| Download a knowledge post template notebook file (Jupyter) | Jupyter Notebook |
| Download a knowledge post template notebook file (R-Markdown) | R-Markdown |
| Download a knowledge post template notebook file (Markdown) | Markdown |
| Write a post in Markdown directly in the Markdown web editor | Web Editor |
| Link a post from elsewhere (e.g. Google Docs) | Linked Post |
| Write a post with uploaded Jupyter Notebook | Jupyter Notebook Post (Beta) |


User Management (Active Directory)
Creation
Let's walk through the basics of user creation for Active Directory. There are some requirement you must know prior to creation:
| Requirement |
|---|
You must connect to your server via TLS or SSL if you set the the users password (unicodepwd ) attribute |
| You must connect to your server with an account that has permission to create users |
You must set a common name (cn ) for the user |
You must set the unicodePwd attribute as a non-encoded string (more on this below) |
To set the users userAccountControl , it must be set after the user has been saved |
Attributes that are set below can be cased in any manor. They can be
UPPERCASED
,lowercased
,camelCased
, etc. Use whichever casing you prefer to be most readable in your application.
<?php
use LdapRecord\Models\ActiveDirectory\User;
$user = (new User)->inside('ou=Users,dc=local,dc=com');
$user->cn = 'John Doe';
$user->unicodePwd = 'SecretPassword';
$user->samaccountname = 'jdoe';
$user->userPrincipalName =<EMAIL_ADDRESS>$user->save();
// Enable the user.
$user->userAccountControl = 512;
try {
$user->save();
} catch (\LdapRecord\LdapRecordException $e) {
// Failed saving user.
}
It is wise to encapsulate saving your user in a try / catch block, so if it fails you can determine if the cause of failure is due to your domains password policy.
Password Management
Setting Passwords
Utilizing the included LdapRecord\Models\ActiveDirectory\User
model, an attribute
mutator has been added that assists in
the setting and changing of passwords on user objects. Feel free to take a
peek into the source code to see how it all works.
The password string you set on the users unicodePwd
attribute is automatically encoded,
you do not need to encode it yourself. Doing so will cause an error or exception upon
saving the user.
Once you have set a password on a user object, this generates a modification
on the user model equal to a LDAP_MODIFY_BATCH_REPLACE
:
<?php
use LdapRecord\Models\ActiveDirectory\User;
$user = new User();
$user->unicodepwd = 'secret';
$modification = $user->getModifications()[0];
var_dump($modification);
// "attrib" => "unicodepwd"
// "modtype" => 3
// "values" => array:1 [▼
// 0 => ""\x00s\x00e\x00c\x00r\x00e\x00t\x00"\x00"
// ]
As you can see, a batch modification is generated for the user and upon
calling save()
, it will be sent to your LDAP server.
Changing Passwords
To change a users password, you must bind to your LDAP server with a user that has permissions to reset passwords, or bind as the user whose password you are trying to change.
There are some prerequisites you must know for changing passwords:
| Requirement |
|---|
| You must provide the correct users old password |
| You must provide a new password that abides by your password policy, such as history, complexity, and length |
You must set the unicodepwd attribute with an array containing two (2) values (old & new password) |
Let's walk through an example:
You must use a try / catch block upon saving. An
LdapRecord\LdapRecordException
will always be thrown when an incorrect old password is given, or the new password does not abide by your password policy.
<?php
use LdapRecord\LdapRecordException;
use LdapRecord\Models\ActiveDirectory\User;
$user = User::find('cn=John Doe,ou=Users,dc=local,dc=com');
$user->unicodepwd = ['old-password', 'new-password'];
try {
$user->save();
// User password changed!
} catch (LdapRecordException $ex) {
// Failed changing password.
$connection = $user->getConnection();
// Get the last LDAP error to determine the cause of failure.
$error = $connection->getLdapConnection()->getDetailedError();
echo $error->getErrorCode();
echo $error->getErrorMessage();
echo $error->getDiagnosticMessage();
}
Resetting Passwords
To reset a users password, you must be bound to your LDAP directory with a user whom has permission to do so on your directory.
You can perform a password reset by simply setting the users unicodepwd
attribute as a string,
and then calling the save()
method, similarly to how it is done during user creation:
<?php
use LdapRecord\LdapRecordException;
use LdapRecord\Models\ActiveDirectory\User;
$user = User::find('cn=John Doe,ou=Users,dc=local,dc=com');
$user->unicodepwd = 'new-password';
try {
$user->save();
// User password reset!
} catch (LdapRecordException $ex) {
// Failed resetting password.
$connection = $user->getConnection();
// Get the last LDAP error to determine the cause of failure.
$error = $connection->getLdapConnection()->getDetailedError();
echo $error->getErrorCode();
echo $error->getErrorMessage();
echo $error->getDiagnosticMessage();
}
Password Policy Errors
Active Directory will return diagnostic error codes when a password modification fails.
To determine the cause, you can check this diagnostic message to see if it contains any of the following codes:
| Code | Meaning |
|---|---|
525 |
User not found |
52e |
Invalid credentials |
530 |
Not permitted to logon at this time |
531 |
Not permitted to logon at this workstation |
532 |
Password expired |
533 |
Account disabled |
701 |
Account expired |
773 |
User must reset password |
775 |
User account locked |
<?php
use LdapRecord\LdapRecordException;
use LdapRecord\Models\ActiveDirectory\User;
$user = User::find('cn=John Doe,ou=Users,dc=local,dc=com');
$user->unicodepwd = ['old-password', 'new-password'];
try {
$user->save();
// User password changed!
} catch (LdapRecordException $ex) {
// Failed resetting password.
$connection = $user->getConnection();
// Get the last LDAP error to determine the cause of failure.
$error = $connection->getLdapConnection()->getDetailedError();
echo $error->getErrorCode(); // 49
echo $error->getErrorMessage(); // 'Invalid credentials'
echo $error->getDiagnosticMessage(); // '80090308: LdapErr: DSID-0C09042A, comment: AcceptSecurityContext error, data 52e, v3839'
if (strpos($error->getDiagnosticMessage(), '52e')) {
// This is an invalid credentials error.
}
}
User Account Control
A users userAccountControl
attribute stores an integer value.
This integer value contains the sums of various integer flags. These flags control the accessibility and behaviour of an Active Directory user account, such as account disablement, password expiry, the ability to change passwords, and more.
For example, setting a users userAccountControl
to 512
would mean that
the user account is a default account type that represents a typical user.
Setting it to 2
, would mean the account is disabled.
Combining both to 514
(512 + 2 = 514
) would mean the users
account is a typical user account, that is also disabled.
Usage
You can manipulate a users userAccountControl
manually by simply setting the
userAccountControl
property on an existing user using the raw integer value,
or you can use the account control builder LdapRecord\Models\Attributes\AccountControl
:
<?php
use LdapRecord\Models\ActiveDirectory\User;
use LdapRecord\Models\Attributes\AccountControl;
$user = User::find('cn=John Doe,ou=Users,dc=local,dc=com');
// Setting the UAC value manually:
$uac = 512; // Normal, enabled account.
// Or, using the UAC builder:
$uac = (new AccountControl)->accountIsNormal();
$user->userAccountControl = $uac;
$user->save();
Using the AccountControl
builder, methods called will automatically sum the integer value.
For example, let's set an account control for a user with the following controls:
- The user account is normal
- The user account password does not expire
- The user account password cannot be changed
$user = User::find('cn=John Doe,ou=Users,dc=local,dc=com');
$uac = new AccountControl();
$uac->accountIsNormal();
$uac->passwordDoesNotExpire();
$uac->passwordCannotBeChanged();
$user->userAccountControl = $uac;
$user->save();
The AccountControl
builder also allows you to determine which flags are set.
This can be done with the has
and doesntHave
methods.
Create an AccountControl
with the users userAccountControl
value in the constructor:
$user = User::find('cn=John Doe,ou=Users,dc=local,dc=com');
$uac = new AccountControl(
$user->getFirstAttribute('userAccountControl')
);
if ($uac->has(AccountControl::LOCKOUT)) {
// This account is locked out.
}
if ($uac->doesntHave(AccountControl::LOCKOUT)) {
// The account is not locked out.
}
Available Constants
The Account Control builder has constants for every possible value:
| Constant | Value |
|---|---|
AccountControl::SCRIPT |
1 |
AccountControl::ACCOUNTDISABLE |
2 |
AccountControl::HOMEDIR_REQUIRED |
8 |
AccountControl::LOCKOUT |
16 |
AccountControl::PASSWD_NOTREQD |
32 |
AccountControl::PASSWD_CANT_CHANGE |
64 |
AccountControl::ENCRYPTED_TEXT_PWD_ALLOWED |
128 |
AccountControl::TEMP_DUPLICATE_ACCOUNT |
256 |
AccountControl::NORMAL_ACCOUNT |
512 |
AccountControl::INTERDOMAIN_TRUST_ACCOUNT |
2048 |
AccountControl::WORKSTATION_TRUST_ACCOUNT |
4096 |
AccountControl::SERVER_TRUST_ACCOUNT |
8192 |
AccountControl::DONT_EXPIRE_PASSWORD |
65536 |
AccountControl::MNS_LOGON_ACCOUNT |
131072 |
AccountControl::SMARTCARD_REQUIRED |
262144 |
AccountControl::TRUSTED_FOR_DELEGATION |
524288 |
AccountControl::NOT_DELEGATED |
1048576 |
AccountControl::USE_DES_KEY_ONLY |
2097152 |
AccountControl::DONT_REQ_PREAUTH |
4194304 |
AccountControl::PASSWORD_EXPIRED |
8388608 |
AccountControl::TRUSTED_TO_AUTH_FOR_DELEGATION |
16777216 |
AccountControl::PARTIAL_SECRETS_ACCOUNT |
67108864 |
Available Methods
The Account Control builder has methods to apply every possible value:
| Method | Constant Applied |
|---|---|
AccountControl::runLoginScript() |
AccountControl::SCRIPT |
AccountControl::accountIsDisabled() |
AccountControl::ACCOUNTDISABLE |
AccountControl::homeFolderIsRequired() |
AccountControl::HOMEDIR_REQUIRED |
AccountControl::accountIsLocked() |
AccountControl::LOCKOUT |
AccountControl::passwordIsNotRequired() |
AccountControl::PASSWD_NOTREQD |
AccountControl::passwordCannotBeChanged() |
AccountControl::PASSWD_CANT_CHANGE |
AccountControl::allowEncryptedTextPassword() |
AccountControl::ENCRYPTED_TEXT_PWD_ALLOWED |
AccountControl::accountIsTemporary() |
AccountControl::TEMP_DUPLICATE_ACCOUNT |
AccountControl::accountIsNormal() |
AccountControl::NORMAL_ACCOUNT |
AccountControl::accountIsForInterdomain() |
AccountControl::INTERDOMAIN_TRUST_ACCOUNT |
AccountControl::accountIsForWorkstation() |
AccountControl::WORKSTATION_TRUST_ACCOUNT |
AccountControl::accountIsForServer() |
AccountControl::SERVER_TRUST_ACCOUNT |
AccountControl::passwordDoesNotExpire() |
AccountControl::DONT_EXPIRE_PASSWORD |
AccountControl::accountIsMnsLogon() |
AccountControl::MNS_LOGON_ACCOUNT |
AccountControl::accountRequiresSmartCard() |
AccountControl::SMARTCARD_REQUIRED |
AccountControl::trustForDelegation() |
AccountControl::TRUSTED_FOR_DELEGATION |
AccountControl::doNotTrustForDelegation() |
AccountControl::NOT_DELEGATED |
AccountControl::useDesKeyOnly() |
AccountControl::USE_DES_KEY_ONLY |
AccountControl::accountDoesNotRequirePreAuth() |
AccountControl::DONT_REQ_PREAUTH |
AccountControl::passwordIsExpired() |
AccountControl::PASSWORD_EXPIRED |
AccountControl::trustToAuthForDelegation() |
AccountControl::TRUSTED_TO_AUTH_FOR_DELEGATION |
AccountControl::accountIsReadOnly() |
AccountControl::PARTIAL_SECRETS_ACCOUNT |
Group Management
If you are utilizing the included LdapRecord\Models\ActiveDirectory\User
model, the
groups()
relationship exists for easily removing / adding groups to users.
Getting Groups
To get the groups that a user is a member of, call the groups()
relationship method.
This will return the immediate groups that the user is a member of:
<?php
use LdapRecord\Models\ActiveDirectory\User;
$user = User::find('cn=John Doe,ou=Users,dc=local,dc=com');
// Get immediate groups the user is apart of:
$groups = $user->groups()->get();
foreach ($groups as $group) {
echo $group->getName();
}
You may also want to retrieve groups that are members of groups that the user is apart of. This is called a recursive relationship query.
To retrieve groups of groups, call the recursive()
method following the groups()
relation call:
<?php
use LdapRecord\Models\ActiveDirectory\User;
$user = User::find('cn=John Doe,ou=Users,dc=local,dc=com');
// Get nested groups the user is apart of:
$groups = $user->groups()->recursive()->get();
foreach ($groups as $group) {
echo $group->getName();
}
Filtering Groups
Relations in LdapRecord act as query builders, so you can chain query methods on the groups()
relation itself:
<?php
use LdapRecord\Models\ActiveDirectory\User;
$user = User::find('cn=John Doe,ou=Users,dc=local,dc=com');
// Get all groups the user is apart of that contain 'Accounting':
$groups = $user->groups()->whereContains('cn', 'Accounting')->get();
// Get all groups the user is apart of that are members of the 'Office' group:
$groups = $user->groups()->whereMemberOf('cn=Office,ou=Groups,dc=local,dc=com')->get();
Checking Existence
To check if a user is a member of any group, call the exists()
method on the groups()
relationship:
<?php
use LdapRecord\Models\ActiveDirectory\User;
$user = User::find('cn=John Doe,ou=Users,dc=local,dc=com');
if ($user->groups()->exists()) {
// The user is a member of at least one group.
}
To check if a user is an immediate member of a specific group, pass a model into the exists()
method:
<?php
use LdapRecord\Models\ActiveDirectory\User;
use LdapRecord\Models\ActiveDirectory\Group;
$group = Group::find('cn=Accounting,dc=local,dc=com');
$user = User::find('cn=John Doe,ou=Users,dc=local,dc=com');
if ($user->groups()->exists($group)) {
// The user is an immediate member of the 'Accounting' group.
}
To check if a user is a member of a group that could be nested in a sub-group, call
the recursive()
method before calling exists()
:
<?php
use LdapRecord\Models\ActiveDirectory\User;
use LdapRecord\Models\ActiveDirectory\Group;
$group = Group::find('cn=Accounting,dc=local,dc=com');
$user = User::find('cn=John Doe,ou=Users,dc=local,dc=com');
if ($user->groups()->recursive()->exists($group)) {
// The user is a member of the 'Accounting' group.
}
Adding Groups
To add groups to a user, call the groups()
relationship method, then attach()
:
<?php
use LdapRecord\Models\ActiveDirectory\User;
use LdapRecord\Models\ActiveDirectory\Group;
$group = Group::findOrFail('cn=Accounting,ou=Groups,dc=local,dc=com');
$user = User::find('cn=John Doe,ou=Users,dc=local,dc=com');
if ($user->groups()->attach($group)) {
// Successfully added the group to the user.
}
Removing Groups
To remove groups on user, call the groups()
relationship method, then detach()
:
<?php
use LdapRecord\Models\ActiveDirectory\User;
use LdapRecord\Models\ActiveDirectory\Group;
$group = Group::findOrFail('cn=Accounting,ou=Groups,dc=local,dc=com');
$user = User::find('cn=John Doe,ou=Users,dc=local,dc=com');
if ($user->groups()->detach($group)) {
// Successfully removed the group from the user.
}
The
detach()
method will returntrue
if the user is already not apart of the given group. This does not indicate that the user was previously a member.
You may want to locate groups on the user prior removal to ensure they are a member:
<?php
use LdapRecord\Models\ActiveDirectory\User;
$user = User::find('cn=John Doe,ou=Users,dc=local,dc=com');
$group = $user->groups()->first();
if ($group && $user->groups()->detach($group)) {
// Successfully removed the first group from the user.
}


Use the arrow keys to change direction.Try not to run into the walls or the snake's tail.
Games Played:
0
High Score:
Current Score:
Level 1


|
Connected Home over IP
672cf7684
|
#include <transport/raw/TCP.h>
Public Member Functions | |
| TCP () | |
| Public Member Functions inherited from chip::Transport::TCPBase | |
| TCPBase (ActiveConnectionState *activeConnectionsBuffer, size_t bufferSize, PendingPacket *packetBuffers, size_t packetsBuffersSize) | |
| ~TCPBase () override | |
| CHIP_ERROR | Init (TcpListenParameters ¶ms) |
| Initialize a TCP transport on a given port. More... | |
| void | Close () override |
| Close the open endpoint without destroying the object. More... | |
| CHIP_ERROR | SendMessage (const PacketHeader &header, const PeerAddress &address, System::PacketBufferHandle msgBuf) override |
| Send a message to the specified target. More... | |
| void | Disconnect (const PeerAddress &address) override |
| Handle disconnection from the specified peer if currently connected to it. More... | |
| bool | CanSendToPeer (const PeerAddress &address) override |
| Determine if this transport can SendMessage to the specified peer address. More... | |
| bool | HasActiveConnections () const |
| Helper method to determine if IO processing is still required for a TCP transport before everything is cleaned up (socket closing is async, so after calling 'Close' on the transport, some time may be needed to actually be able to close.) More... | |
| void | CloseActiveConnections () |
| Close all active connections. More... | |
| Public Member Functions inherited from chip::Transport::Base | |
| virtual | ~Base () |
| void | SetDelegate (RawTransportDelegate *delegate) |
| Sets the delegate of the transport. More... | |
Friends | |
| class | TCPTest |
Additional Inherited Members | |
| Protected Member Functions inherited from chip::Transport::Base | |
| void | HandleMessageReceived (const PacketHeader &header, const PeerAddress &source, System::PacketBufferHandle &&buffer) |
| Method used by subclasses to notify that a packet has been received after any associated headers have been decoded. More... | |
| Protected Attributes inherited from chip::Transport::Base | |
| RawTransportDelegate * | mDelegate |
|
inline |
|
friend |


Aston Villa and Manchester United transfer target has 'release clause' - if this happens
Latest Aston Villa transfer news includes claims over Leeds United midfielder Kalvin Phillips who has recently been linked with Manchester United
by Shane Ireland, https://www.facebook.com/ShaneIrelandJournalistKalvin Phillips has a release clause in his contract allowing him to leave if Leeds United don’t win promotion, according to Noel Whelan.
Phillips, 23, has been linked with a move to the Premier League after becoming a mainstay of Marcelo Bielsa’s side at Elland Road.
Aston Villa were firmly interested during the summer transfer window when Phillips was a leading target after the play-off triumph in May.
But they couldn’t agree a fee with Leeds, who rejected bids, and the midfielder stayed put for another shot at promotion with his boyhood club.
Since then, Manchester United have also been credited with an interest amid reports they have been scouting Phillips this season.
Aston Villa latest
This is your round up of all things Villa; the one-stop shop that will keep you updated on the latest goings on at the club and beyond.
Transfers, injuries, match-days and managers, we’ve got you covered.
We’ll be bringing the very latest on the Villains throughout the week and around the clock.
Make sure you keep yourself updated with our handy daily catch up...
Must reads...
- Villa fans make Chelsea transfer demand
- 'Worried' fans react to major announcement
- Former Villan reveals Newcastle bombshell
- Morelos saga takes new twist
For all your latest AVFC news, opinion, analysis and transfer gossip, click here
You can follow our Villa reporter Ashley Preece on Twitter, and his email address is<EMAIL_ADDRESS>He signed a new contract in August and Whelan has claimed that as part of that deal, suitors can activate a get-out clause in the summer if Leeds fail to return to the Premier League.
“Kalvin’s contract is based on promotion," the former Whites player told Football Insider. "He has a release clause allowing him to go for a set price – more than £20million – if we don’t go up.”
Leeds are believed to have turned down offers in the region of £20m for Phillips in the summer and Villa looked elsewhere to add to their options in the middle of the park.
Marvelous Nakamba and Douglas Luiz arrived into Dean Smith's squad as part of a big-money spending spree.


Chapter 1 Data Manipulation using dplyr
Learning Objectives
- Select columns in a data frame with the
dplyr
functionselect
.- Select rows in a data frame according to filtering conditions with the
dplyr
functionfilter
.- Direct the output of one
dplyr
function to the input of another function with the ‘pipe’ operator%>%
.- Add new columns to a data frame that are functions of existing columns with
mutate
.- Understand the split-apply-combine concept for data analysis.
- Use
summarize
,group_by
, andtally
to split a data frame into groups of observations, apply a summary statistics for each group, and then combine the results.
We will be working a small subset of the data from the Stanford Open Policing Project. It contains information about traffic stops for blacks and whites in the state of Mississippi during January 2013 to mid-July of 2016.
Let’s begin with loading our sample data into a data frame.
Manipulation of dataframes is a common task when you start exploring your data. We might select certain observations (rows) or variables (columns), group the data by a certain variable(s), or calculate summary statistics.
If we were interested in the mean age of the driver in different counties we can do this using the normal base R operations:
#> [1] 31.8002
#> [1] 34.66915
#> [1] 37.05759
Bracket subsetting is handy, but it can be cumbersome and difficult to read, especially for complicated operations. Furthermore, there is a fair amount of repetition. Repeating yourself will cost you time, both now and later, and potentially introduce some nasty bugs.
dplyr
is a package for making tabular data manipulation easier.
Brief recap: Packages in R are sets of additional functions that let you do more stuff. Functions like
str()
ordata.frame()
, come built into R; packages give you access to more of them. Before you use a package for the first time you need to install it on your machine, and then you should import it in every subsequent R session when you need it.
If you haven’t, please installe the tidyverse
package.
tidyverse
is an “umbrella-package” that installs a series of packages useful for data analysis which work together well. Some of them are considered core packages (among them tidyr
, dplyr
, ggplot2
), because you are likely to use them in almost every analysis. Other packages, like lubridate
(to work wiht dates) or haven
(for SPSS, Stata, and SAS data) that you are likely to use not for every analysis are also installed.
If you type the following command, it will load the core tidyverse
packages.
If you need to use functions from tidyverse
packages other than the core packages, you will need to load them separately.
1.1 What is dplyr
?
dplyr
is one part of a larger tidyverse
that enables you to work
with data in tidy data formats. “Tidy datasets are easy to manipulate, model and visualise, and have a specific structure: each variable is a column, each observation is a row, and each type of observational unit is a table.” (From Wickham, H. (2014): Tidy Data https://www.jstatsoft.org/article/view/v059i10)
The package dplyr
provides convenient tools for the most common data manipulation
tasks. It is built to work directly with data frames, with many common tasks
optimized by being written in a compiled language (C++). An additional feature is the
ability to work directly with data stored in an external database. The benefits of
doing this are that the data can be managed natively in a relational database,
queries can be conducted on that database, and only the results of the query are
returned.
This addresses a common problem with R in that all operations are conducted in-memory and thus the amount of data you can work with is limited by available memory. The database connections essentially remove that limitation in that you can have a database of many 100s GB, conduct queries on it directly, and pull back into R only what you need for analysis.
To learn more about dplyr
after the workshop, you may want to check out the handy data transformation with dplyr
cheatsheet.
1.2 Subsetting columns and rows
To select columns of a
data frame with dplyr
, use select()
. The first argument to this function is the data
frame (trafficstops
), and the subsequent arguments are the columns to keep.
#> police_department officer_id driver_race
#> 1 Mississippi Highway Patrol J042 Black
#> 2 Mississippi Highway Patrol B026 Black
#> 3 Mississippi Highway Patrol M009 Black
#> 4 Mississippi Highway Patrol K035 White
#> 5 Mississippi Highway Patrol D028 White
#> 6 Mississippi Highway Patrol K023 White
It is worth knowing that dplyr
comes with a number of “select helpers”, which are functions that allow you to select columns based on their names. For example:
#> driver_gender driver_birthdate driver_race driver_age
#> 1 male 1950-06-14 Black 63
#> 2 male 1967-04-06 Black 46
#> 3 male 1974-04-15 Black 39
#> 4 male 1981-03-23 White 32
#> 5 male 1992-08-03 White 20
#> 6 female 1960-05-02 White 53
To choose rows based on specific criteria, use filter()
:
#> id state stop_date county_name county_fips
#> 1 MS-2013-00252 MS 2013-01-02 Yazoo County 28163
#> 2 MS-2013-00253 MS 2013-01-02 Yazoo County 28163
#> 3 MS-2013-00254 MS 2013-01-02 Yazoo County 28163
#> 4 MS-2013-00331 MS 2013-01-02 Yazoo County 28163
#> 5 MS-2013-00350 MS 2013-01-02 Yazoo County 28163
#> 6 MS-2013-00426 MS 2013-01-03 Yazoo County 28163
#> police_department driver_gender driver_birthdate driver_race
#> 1 Mississippi Highway Patrol male 1950-05-04 Black
#> 2 Mississippi Highway Patrol female 1967-05-29 Black
#> 3 Mississippi Highway Patrol male 1986-12-21 Black
#> 4 Mississippi Highway Patrol female 1986-02-01 Black
#> 5 Mississippi Highway Patrol male 1994-11-21 White
#> 6 Mississippi Highway Patrol male 1994-02-24 White
#> violation_raw officer_id
#> 1 Speeding - Regulated or posted speed limit and actual speed C037
#> 2 Speeding - Regulated or posted speed limit and actual speed C011
#> 3 Speeding - Regulated or posted speed limit and actual speed C011
#> 4 Speeding - Regulated or posted speed limit and actual speed C037
#> 5 Speeding - Regulated or posted speed limit and actual speed C037
#> 6 Speeding - Regulated or posted speed limit and actual speed C014
#> driver_age violation
#> 1 63 Speeding
#> 2 46 Speeding
#> 3 26 Speeding
#> 4 27 Speeding
#> 5 18 Speeding
#> 6 19 Speeding
Here are some other ways to select rows:
- select certain rows by row number:
slice(trafficstops, 1:3) # rows 1-3
- select random rows:
sample_n(trafficstops, 5) # number of rows to select
sample_frac(trafficstops, .01) # fraction of rows to select
To sort rows by variables use the arrange
function: arrange(trafficstops, county_name, stop_date)
#> id state stop_date county_name county_fips
#> 1 MS-2013-07659 MS 2013-02-09 Adams County 28001
#> 2 MS-2013-11819 MS 2013-03-02 Adams County 28001
#> 3 MS-2013-14647 MS 2013-03-16 Adams County 28001
#> 4 MS-2013-15430 MS 2013-03-20 Adams County 28001
#> 5 MS-2013-18581 MS 2013-04-06 Adams County 28001
#> 6 MS-2013-20016 MS 2013-04-13 Adams County 28001
#> police_department driver_gender driver_birthdate driver_race
#> 1 Mississippi Highway Patrol male 1989-06-12 Black
#> 2 Mississippi Highway Patrol female 1974-10-16 Black
#> 3 Mississippi Highway Patrol female 1977-07-15 Black
#> 4 Mississippi Highway Patrol female 1991-06-15 Black
#> 5 Mississippi Highway Patrol female 1980-04-18 White
#> 6 Mississippi Highway Patrol female 1996-01-14 Black
#> violation_raw officer_id
#> 1 Speeding - Regulated or posted speed limit and actual speed M004
#> 2 Driving while license suspended M042
#> 3 Speeding - Regulated or posted speed limit and actual speed M049
#> 4 Failure to maintain required liability insurance M049
#> 5 Failure to maintain required liability insurance M010
#> 6 Speeding - Regulated or posted speed limit and actual speed M024
#> driver_age violation
#> 1 24 Speeding
#> 2 38 License-Permit-Insurance
#> 3 36 Speeding
#> 4 22 License-Permit-Insurance
#> 5 33 License-Permit-Insurance
#> 6 17 Speeding
1.3 Pipes
What if you wanted to filter and select on the same data? For example, lets find drivers over 85 years and only keep the violation and gender columns. There are three ways to do this: use intermediate steps, nested functions, or pipes.
- Intermediate steps:
With intermediate steps, you essentially create a temporary data frame and use that as input to the next function. This can clutter up your workspace with lots of objects.
- Nested functions
You can also nest functions (i.e. one function inside of another). This is handy, but can be difficult to read if too many functions are nested as things are evaluated from the inside out.
- Pipes!
The last option, pipes, are a fairly recent addition to R. Pipes let you take
the output of one function and send it directly to the next, which is useful
when you need to do many things to the same dataset. Pipes in R look like
%>%
and are made available via the magrittr
package, installed automatically
with dplyr
. If you use RStudio, you can type the pipe with Ctrl
+ Shift + M if you have a PC or Cmd +
Shift + M if you have a Mac.
In the above, we use the pipe to send the trafficstops
dataset first through
filter()
to keep rows where driver_race
is Black, then through select()
to keep only the officer_id
and stop_date
columns. Since %>%
takes
the object on its left and passes it as the first argument to the function on
its right, we don’t need to explicitly include it as an argument to the
filter()
and select()
functions anymore.
If we wanted to create a new object with this smaller version of the data, we could do so by assigning it a new name:
senior_drivers <- trafficstops %>%
filter(driver_age > 85) %>%
select(violation_raw, driver_gender, driver_race)
senior_drivers
#> violation_raw
#> 1 Seat belt not used properly as required
#> 2 Speeding - Regulated or posted speed limit and actual speed
#> 3 Seat belt not used properly as required
#> driver_gender driver_race
#> 1 male White
#> 2 male White
#> 3 male Black
Note that the final data frame is the leftmost part of this expression.
Challenge
Using pipes, subset the
trafficstops
data to include stops in Tunica County only and retain the columnsstop_date
,driver_age
, andviolation_raw
. Bonus: sort the table by driver age.
1.4 Add new columns
Frequently you’ll want to create new columns based on the values in existing columns. For this we’ll use mutate()
.
To create a new column with the year the driver was born we can extract the first 4 elements of the string that represents the driver_birthdate
and add it to the data frame like this:
The new and edited columns will not permanently be added to the existing data frame – unless we explicitly save the output.
We can als use mutate
to reassign values to an existing column. So here is an alternative to do the same as above, but here we first will use the lubridate
library, which is installed with tidyverse
to convert our string to an actual date format. We will then use the year()
function to extract the year. All of this can happen in the same call to mutate()
.
library(lubridate)
trafficstops %>%
mutate(birth_date = ymd(driver_birthdate),
birth_year = year(driver_birthdate))
If this runs off your screen and you just want to see the first few rows, you
can use a pipe to view the head()
of the data. (Pipes work with non-dplyr
functions, too, as long as the dplyr
or magrittr
package is loaded). When piping into a function with no additional arguments, you can call the
function with or without parentheses (e.g. head
or head()
). (I like to add the parentheses to remind myself that it is a function and not a variable.)
trafficstops %>%
mutate(birth_date = ymd(driver_birthdate),
birth_year = year(driver_birthdate)) %>%
head()
We can keep adding columns like this:
trafficstops %>%
mutate(birth_date = ymd(driver_birthdate),
birth_year = year(driver_birthdate),
birth_cohort = round(birth_year/10)*10) %>%
head()
We are beginning to see the power of piping. Here is a slightly expanded example, where we select the column birth_cohort
that we have created and send it to plot:
trafficstops %>%
mutate(birth_date = ymd(driver_birthdate),
birth_year = year(driver_birthdate),
birth_cohort = round(birth_year/10)*10,
birth_cohort = factor(birth_cohort)) %>%
select(birth_cohort) %>%
plot()
Challenge
Create a new data frame from the
trafficstops
data that meets the following criteria: contains only theviolation_raw
column for female drivers of age 50 that were stopped on a Sunday. For this add a new column to your data frame calledweekday_of_stop
containing the number of the weekday when the stop occurred. Use thewday()
function fromlubridate
(Sunday = 1).Think about how the commands should be ordered to produce this data frame!
1.5 What is split-apply-combine?
Many data analysis tasks can be approached using the split-apply-combine paradigm: split the data into groups, apply some analysis to each group, and then combine the results.
dplyr
makes this very easy through the use of the
group_by()
function.
group_by()
is often used together with summarize()
, which collapses each
group into a single-row summary of that group. group_by()
takes as arguments
the column names that contain the categorical variables for which you want
to calculate the summary statistics. So to view the mean age for black and white drivers:
#> Warning: Factor `driver_race` contains implicit NA, consider using
#> `forcats::fct_explicit_na`
#> # A tibble: 3 x 2
#> driver_race mean_age
#> <fct> <dbl>
#> 1 Black 34.2
#> 2 White 36.2
#> 3 <NA> 34.5
So, back to the beginning of the chapter, where we tried to calculate the mean age of the driver for different counties, how would we do this? Like this:
You can also group by multiple columns:
trafficstops %>%
group_by(county_name, driver_race) %>%
summarize(mean_age = mean(driver_age, na.rm=TRUE))
#> Warning: Factor `driver_race` contains implicit NA, consider using
#> `forcats::fct_explicit_na`
#> # A tibble: 178 x 3
#> # Groups: county_name [82]
#> county_name driver_race mean_age
#> <fct> <fct> <dbl>
#> 1 Adams County Black 36.2
#> 2 Adams County White 40.0
#> 3 Alcorn County Black 34.6
#> 4 Alcorn County White 33.6
#> 5 Amite County Black 37.5
#> 6 Amite County White 42.1
#> 7 Amite County <NA> 24
#> 8 Attala County Black 36.4
#> 9 Attala County White 38.6
#> 10 Benton County Black 34.7
#> # … with 168 more rows
If we wanted to remove the line with NA
we could insert a filter()
in the chain:
trafficstops %>%
filter(!is.na(driver_race)) %>%
group_by(county_name, driver_race) %>%
summarize(mean_age = mean(driver_age, na.rm=TRUE))
#> # A tibble: 163 x 3
#> # Groups: county_name [82]
#> county_name driver_race mean_age
#> <fct> <fct> <dbl>
#> 1 Adams County Black 36.2
#> 2 Adams County White 40.0
#> 3 Alcorn County Black 34.6
#> 4 Alcorn County White 33.6
#> 5 Amite County Black 37.5
#> 6 Amite County White 42.1
#> 7 Attala County Black 36.4
#> 8 Attala County White 38.6
#> 9 Benton County Black 34.7
#> 10 Benton County White 32.0
#> # … with 153 more rows
Recall that is.na()
is a function that determines whether something is an NA
. The !
symbol negates the result, so we’re asking for everything that is not an NA
.
You may have noticed that the output from these calls looks a little different. That’s because dplyr
has changed our data.frame
object
to an object of class tbl_df
, also known as a “tibble”. Tibble’s data
structure is very similar to a data frame. For our purposes the only differences
are that (1) columns of class character
are never converted into
factors, and (2) in addition to displaying the data type of each column under its name, it only prints the first few rows of data and only as many columns as fit on one screen. If we wanted to print all columns we can use the print command, and set the width
parameter to Inf
. To print the first 6 rows for example we would do this: print(my_tibble, n=6, width=Inf)
.
Once the data are grouped, you can also summarize multiple variables at the same time (and not necessarily on the same variable). For instance, we could add a column indicating the minimum age in each group (i.e. county):
trafficstops %>%
filter(!is.na(driver_race)) %>%
group_by(county_name, driver_race) %>%
summarize(mean_age = mean(driver_age, na.rm=TRUE),
min_age = min(driver_age, na.rm=TRUE))
#> # A tibble: 163 x 4
#> # Groups: county_name [82]
#> county_name driver_race mean_age min_age
#> <fct> <fct> <dbl> <int>
#> 1 Adams County Black 36.2 16
#> 2 Adams County White 40.0 16
#> 3 Alcorn County Black 34.6 17
#> 4 Alcorn County White 33.6 15
#> 5 Amite County Black 37.5 17
#> 6 Amite County White 42.1 15
#> 7 Attala County Black 36.4 8
#> 8 Attala County White 38.6 15
#> 9 Benton County Black 34.7 18
#> 10 Benton County White 32.0 18
#> # … with 153 more rows
1.6 Tallying
When working with data, it is also common to want to know the number of
observations found for each factor or combination of factors. For this, dplyr
provides tally()
. For example, if we wanted to see how many traffic stops each officer recorded we would do:
We can optionally sort the results in descending order by adding sort=TRUE
:
Here, tally()
is the action applied to the groups created by group_by()
and counts the total number of records for each category.
Alternatives:
trafficstops %>%
group_by(officer_id) %>%
summarize(n = n()) # n() is useful when count is needed for a calculation
#> Warning: Factor `officer_id` contains implicit NA, consider using
#> `forcats::fct_explicit_na`
#> Warning: Factor `officer_id` contains implicit NA, consider using
#> `forcats::fct_explicit_na`
Challenge
Which 5 counties were the ones with the most stops in 2013? Hint: use the year() function from lubridate.
1.7 Joining two tables
<<<<<<< HEAD It is not uncommon that we have our data spread out in different tables and need to bring those together for analysis. In this example we will combine the numbers of trafficstops for black and white drivers per county together with the numbers of the black and white total population for these counties. The population data are the estimated values of the 5 year average from the 2011-2015 American Community Survey (ACS): ======= It is not uncommon that we have our data spread out in different tables and need to bring those together for analysis. In this example we will combine the numbers of trafficstops for black and white drivers per county together with the numbers of the black and white total population for these counties. Toe population data are the estimated values of the 5 year average from the 2011-2015 American Community Survey (ACS): >>>>>>> 1aab4b954ad0f82372da78447cb5bc64fdc759fe
#> County FIPS black_pop white_pop bw_pop
#> 1 Jones County 28067 19711 47154 66865
#> 2 Lauderdale County 28075 33893 43482 77375
#> 3 Pike County 28113 21028 18282 39310
#> 4 Hancock County 28045 4172 39686 43858
#> 5 Holmes County 28051 15498 3105 18603
#> 6 Jackson County 28059 30704<PHONE_NUMBER>90
In a first step we will use a prevous dplyr
command to count all the trafficstops per county.
#> # A tibble: 82 x 2
#> county_name n_stops
#> <fct> <int>
#> 1 Adams County 942
#> 2 Alcorn County 3345
#> 3 Amite County 2921
#> 4 Attala County 4203
#> 5 Benton County 214
#> 6 Bolivar County 4526
#> 7 Calhoun County 1658
#> 8 Carroll County 1788
#> 9 Chickasaw County 3869
#> 10 Choctaw County 613
#> # … with 72 more rows
We will then pipe this into our next operation where we bring the two tables together. We will use left_join
, which returns all rows from the left table, and all columns from the left and the right table. As unique ID, which uniquely identifies the corresponding records in each table we use the County Names.
trafficstops %>%
group_by(county_name) %>%
summarise(n_stops = n()) %>%
left_join(MS_bw_pop, by = c("county_name" = "County")) %>%
head()
#> # A tibble: 6 x 6
#> county_name n_stops FIPS black_pop white_pop bw_pop
#> <fct> <int> <int> <int> <int> <int>
#> 1 Adams County 942 28001 17757 12856 30613
#> 2 Alcorn County 3345 28003 4281 31563 35844
#> 3 Amite County 2921 28005 5416 7395 12811
#> 4 Attala County 4203 28007 8194 10649 18843
#> 5 Benton County 214 28009 3078 5166 8244
#> 6 Bolivar County 4526 28011 21648 11197 32845
Now we can, for example calculate the percentage of the population that gets stopped in each county.
Challenge
Which county has the highest (lowest) percentage of stopped drivers? Use the snippet from above and pipe into the additional operations to do this.
dplyr
join functions are generally equivalent merge
from the base command, but there are a few advantages:
- rows are kept in existing order
- much faster
- tells you what keys you’re merging by (if you don’t supply)
- also work with database tables.
https://groups.google.com/d/msg/manipulatr/OuAPC4VyfIc/Qnt8mDfq0WwJ
See ?dplyr::join
for all the possible joins.


Enjoy your favourite movie,TV show & video free.
20 Jun,2017
10 October,2010
02 August,2019
Lastest Movies
22 Jun 2019
22 Jun 2017
12 August 2018
9 August 2019
TV Shows
No matter who you are or who you be with, I will always truly completely honestly love you.


An ICO or Initial Coin Offering is the first time a cryptocurrency is offered to the public for purchase and is a method for a company to raise money, in order to develop the technology the currency will support.
* A Cryptocurrency is a store of value and can represent any asset. It can be a currency, represent membership, be a share in a company, etc. *The company that offered the coin for purchase will be the recipient of the funds raise from the ICO. These funds will be used to enable the company to develop their product.
A company will often hold onto a portion of the coins and not offer them to the public for purchase. This means that the company has a personal stake in their products success. If their product is successful and therefore more people begin to use their coin, the price of the coin will go up making their holdings more valuable. However, if the product is unsuccessful, their holdings will lose their value.
ICO's are offered on online cryptocurrency exchanges, making them very easy to purchase from. Sometimes the company will also offer a Pre ICO which will allow the very first people to purchase the coin at a discounted price.
Currently there is very little ICO specific regulation in countries. However, companies that hold ICO's are still bound to the laws and regulations of their country. For example in many countries if a company were to offer an ICO with no intention of fulfilling what they promise, the country regulators may be able to charge them with fraud.


ao1-html-evan.html
First Name:
Esports
Esports is a new form of enterainment for the younger generation, if you haven't heard of it you may think
its quite small, you'd be wrong, its actually huge.
here is a link to a Esports wiki page. link
above is a picture of the CWL Championships main stage.
Call of Duty Esports is a smaller Esport but ones like League of Legands or Dota 2 are massive, each can sell
out a staduim easily.
here's a list of 4 popular Esports currently.
Dota 2
League of Legends
CS:GO
Call of Duty
If you'd like to watch a video talking about a little bit of background to Dota 2 here is a video to watch.
VIDEO


Web scraping
We begin with a discussion on web scraping. The term itself is ambiguous, and could potentially mean anything1, but the gist is that there is something out on the web (or some connected machine) that we want, and we’d like to use R to get it. This section won’t (at least for now) get into lower level utilities such as that provided by httr or Rcurl, though some packages will be using them under the hood. Instead, focus will be on higher-level approaches with an eye toward common tasks.
As a starting point, open a browser and go to a website of your preference. For most browsers, Ctrl+U will open up the underlying html file. If you’re on a typical webpage it will almost certainly look like a mess of html, JavaScript, XML and possibly other things. Simpler pages are more easily discernible, while more complex/secure pages are not. The take home message is that what you want is represented by something in there, and you’ll have to know something about that structure in order to get it.
Unfortunately, a lot of web design is quite poor2, which will make your job difficult. Even when the sole purpose of a site is to provide data, you can almost be sure that the simplest/most flexible path will not be taken to do so. Thankfully you can use R and possibly other tools to make the process at least somewhat less painful.
Direct data download
One thing to be aware of is that you can get data from the web that are in typical formats just as you would any file on your own machine. For example:
mydata = read.csv('http://somewebsite/data.csv')
I wouldn’t even call this web scraping, just as you wouldn’t if you were getting a file off a network connection. However, if you’re just starting out with R, it’s important to know this is possible. In any case, do not make things more difficult than they need to be- if the file is available just grab it.
Key concepts
The web works in mysterious ways, but you don’t have to know a lot about it to use it or benefit from its data. Many people have had their own websites for years without knowing any html. For our purposes, it helps to know a little and is actually required, at least so that we know what to look for.
Common elements or tags3 of a webpage include things like div
table
and ul
and body
, and within such things our data will be found. Some common ones include:
div
andspan
: used when other elements are not appropriate but one still wants to define classes, ids etc.p
paragrapha
link (technically ‘anchor’)ul
ol
liststable
tr
td
tables, table rows, table data (cell)h#
i.e.h1
h2
etc. headersimg
images
Consider the following:

 This header denotes a section of text

 This is a paragraph!

 Here is another! This one has a link!

As an example, if we wanted to scrape every paragraph that had the word paragraph in it, we’d need to first get all the

elements and work from there.
Just like in other programming languages, these parts of a webpage have their own class, e.g.

, and this allows even easier access to the objects of specific interest. Take a look at the source for the Wikipedia page for towns in Michigan.
The id is another attribute, or way to note specific types of objects. Unlike classes, which might be used over and over within a page, ids are unique, and thus for web scraping, probably more useful when appropriate to the task. Also, while elements can have multiple classes, they can have only one id. Any particular element might also have different attributes that specify things like style, alignment, color, file source etc.
The sad thing is that attributes are greatly underutilized in general. For example, perhaps you found a page with several html tables of data, one for each year. It would have been simple to add an id = year
to each one, enabling you to grab just the specific year(s) of interest, but it’ll be rare that you’d find a page that goes that far. Part of this could be due to the fact that much content is auto-generated via WYSIWYG editors, but these are not very professional pages if so, which might cause concern for the data given such a source. Take a look at the open.fda.gov site:
It has lots of attributes for practically every element, which would make getting stuff off the page fairly easy, if it didn’t already have an API.
In any case, you’ll at least need to know the tag within which the data or link to it may be found. Classes and ids will help to further drill down the web page content, but often won’t be available. In such a case you might have to grab, for example, all the links, and then use some regular expression to grab only the one you want. See this link for more on relations between attributes and elements.
The basic approach
To summarize the basic approach, we can outline a few steps:
- If a direct link or API is available use it4
- Inspect the page to find the relevant tags within which the content is found
- Start with a base URL
- Use the relevant R packages to parse the base page
- Extract the desired content
Examples
One of the packages that can make scraping easy is rvest, which is modeled after/inspired by the Beautiful Soup module in Python5. As the goal here is to get you quickly started, we won’t inundate you with a lot of packages yet. I recommend getting comfy with rvest and moving to others when needed.
Tables
Wikipedia
A lot of Wikipedia has pages ripe for the scraping, but it also has an API, so that we can use it as an example later as well. Between the page layout and rvest it will be very easy to get the content. So let’s do so. Back to the page of towns in Michigan. Let’s see if we can get the table of towns with their type, county, and population. First things first, we need to get the page.
page = 'https://en.wikipedia.org/wiki/List_of_cities,_villages,_and_townships_in_Michigan'
library(rvest)
towns = read_html(page)
str(towns)
List of 2
$ node:<externalptr>
$ doc :<externalptr>
- attr(*, "class")= chr [1:2] "xml_document" "xml_node"
towns
{xml_document}

[1] \n <meta charset="UTF-8"/>\n <title>List of cities, villages, and townships in Michigan - Wikipedia</titl ...
[2] <body class="mediawiki ltr sitedir-ltr mw-hide-empty-elt ns-0 ns-subject page-List_of_cities_villages_and_townshi ...
The result of read_html is an xml_document class object. For the uninitiated, XML is a markup language (Extensible Markup Language) like HTML, and which allows one to access its parts as nodes in tree6, where parents have children and grandchildren etc. It will require further parsing in order to get what we want, but it was easy enough to snag the page. Let’s look at some of the nodes.
html_nodes(towns, 'ul') %>% head # unordered lists (first few)
{xml_nodeset (6)}
[1]

A
\n
B
\n
C
\n
D ...
[2]

See also
\n
References
\n
<a href="#Exter ...
[3]

Adminis ...
[4]

<a rel="nofollow" class="external text" href="https://web.archive.org/web/20040908085409/http://www.vienn ...
[5]

Cens ...
[6]

% head # links
{xml_nodeset (6)}
[1]
[2] navigation
[3] search
[4] list of references
[5] external links
[6] inline citations
html_nodes(towns, 'table') # tables
{xml_nodeset (5)}
[1]
\n
<td class="mb ...
[2]
\n
Place	\nType	\nCounty	\n2010 Population</th ...
[3] <table role="presentation" class="mbox-small plainlinks sistersitebox" style="border:1px solid #aaa;background-co ...
[4] <table class="nowraplinks collapsible autocollapse navbox-inner" style="border-spacing:0;background:transparent;c ...
[5] <table class="nowraplinks collapsible autocollapse navbox-inner" style="border-spacing:0;background:transparent;c ...
Perhaps now you are getting a sense of what we can possibly extract from this page. It turns out that what we want is a table element, and in particular, the one that is of class wikitable sortable
. This will make things very easy. But what if we didn’t know what the element was? it turns out there are tools like SelectorGadget7 that can be used to inspect the elements of any webpage. SelectorGadget can be added as a browser extension, which would allow you to easily turn it on and off as needed. The following depicts what the Michigan towns looks like when using SelectorGadget and selecting a link.
It highlights everything else in the page that is of a similar type (i.e. other links), as well as provides the css/xml info that we would need to grab that particular item.
As mentioned, we want the wikitable sortable
class table, so lets grab that. The rvest package comes with the html_table function, that will grab any table and attempt to put it into a data.frame object. I only show the first couple because one of the tables comes out pretty messy at the R console.
str(html_table(towns)[1:3])
List of 3
$ :'data.frame': 1 obs. of 2 variables:
..$ X1: logi NA
..$ X2: chr "This article includes a list of references, related reading or external links, but its sources remain unclear because it lacks "| __truncated__
$ :'data.frame': 2185 obs. of 4 variables:
..$ Place : chr [1:2185] "Acme" "Acme Township" "Ada Township" "Adams Township, Arenac County" ...
..$ Type : chr [1:2185] "unincorporated community" "township" "township" "township" ...
..$ County : chr [1:2185] "" "" "Kent" "Arenac" ...
..$ 2010 Population: chr [1:2185] "" "4,375" "13,142" "563" ...
$ :'data.frame': 1 obs. of 2 variables:
..$ X1: logi NA
..$ X2: chr "Wikimedia Commons has media related to Populated places in Michigan."
We know which class object we want though, so we could have used html_nodes to grab only that object and work with it.
html_nodes(towns, '.wikitable.sortable') %>%
html_table() %>%
.[[1]] %>%
head()
Place Type County 2010 Population
1 Acme unincorporated community
2 Acme Township township 4,375
3 Ada Township township Kent 13,142
4 Adams Township, Arenac County township Arenac 563
5 Adams Township, Hillsdale County township Hillsdale 2,493
6 Adams Township, Houghton County township Houghton 2,573
Here the .
in .wikitable.sortable
represents the class/subclass (use #
for ids). In general though, if you know it’s a table, then use html_table. When the table has no class then you’ll have to use a magic number or some other means to grab it, which may not be as robust. Of course, the class itself might change over time also.
So let’s put this all together.
library(stringr)
towns %>%
html_table() %>%
.[[2]] %>%
rename(Population = `2010 Population`) %>%
mutate(Population = strtoi(str_replace_all(Population, ',', '')),
Type = factor(Type)) %>%
filter(!Type %in% c('unincorporated community', 'CDP')) %>%
ggplot(aes(x=Population, fill=Type, color=Type)) +
scale_x_log10() +
geom_density(alpha=.2) +
theme(panel.background=element_rect(fill='transparent', color=NA),
plot.background=element_rect(fill='transparent', color=NA))
I won’t go into details about the rest of the code regarding data processing, as that’s for another workshop. For now it suffices to say that it didn’t take much to go from URL to visualization.
Basketball Reference
As an additional example let’s get some data from basketball-reference.com. The following gets the totals table from the URL8. Issues include header rows after every 20th row of data, and converting all but a few columns from character to numeric.
url = "http://www.basketball-reference.com/leagues/NBA_2016_totals.html"
bball = read_html(url) %>%
html_node("#totals_stats") %>% # grab element with id 'total_stats'
html_table() %>% # the data
filter(Rk != "Rk") %>% # remove header rows
mutate_at(vars(-Player, -Pos, -Tm), as.numeric) # convert to numeric
str(bball)
'data.frame': 578 obs. of 30 variables:
$ Rk : num 1 2 3 4 5 6 7 8 9 10 ...
$ Player: chr "Quincy Acy" "Jordan Adams" "Steven Adams" "Arron Afflalo" ...
$ Pos : chr "PF" "SG" "C" "SG" ...
$ Age : num 25 21 22 30 27 27 30 20 26 34 ...
$ Tm : chr "SAC" "MEM" "OKC" "NYK" ...
$ G : num 59 2 80 71 59 60 74 8 79 64 ...
$ GS : num 29 0 80 57 17 5 74 0 28 57 ...
$ MP : num 876 15 2014 2371 861 ...
$ FG : num 119 2 261 354 150 134 536 5 191 215 ...
$ FGA : num 214 6 426 799 315 ...
$ FG% : num 0.556 0.333 0.613 0.443 0.476 0.596 0.513 0.5 0.516 0.458 ...
$ 3P : num 19 0 0 91 0 0 0 0 0 15 ...
$ 3PA : num 49 1 0 238 1 0 16 0 0 42 ...
$ 3P% : num 0.388 0 NA 0.382 0 NA 0 NA NA 0.357 ...
$ 2P : num 100 2 261 263 150 134 536 5 191 200 ...
$ 2PA : num 165 5 426 561 314 ...
$ 2P% : num 0.606 0.4 0.613 0.469 0.478 0.596 0.521 0.5 0.516 0.468 ...
$ eFG% : num 0.6 0.333 0.613 0.5 0.476 0.596 0.513 0.5 0.516 0.474 ...
$ FT : num 50 3 114 110 52 60 259 0 46 90 ...
$ FTA : num 68 5 196 131 62 84 302 0 73 138 ...
$ FT% : num 0.735 0.6 0.582 0.84 0.839 0.714 0.858 NA 0.63 0.652 ...
$ ORB : num 65 0 219 23 75 86 176 2 162 104 ...
$ DRB : num 123 2 314 243 194 202 456 4 262 192 ...
$ TRB : num 188 2 533 266 269 288 632 6 424 296 ...
$ AST : num 27 3 62 144 31 50 110 0 76 70 ...
$ STL : num 29 3 42 25 19 47 38 1 26 110 ...
$ BLK : num 24 0 89 10 36 68 81 2 42 18 ...
$ TOV : num 27 2 84 82 54 64 99 1 69 78 ...
$ PF : num 103 2 223 142 134 139 151 1 147 175 ...
$ PTS : num 307 7 636 909 352 ...
Text
A lot of times we’ll want to grab text as opposed to tables. See the API chapter for an example, and the html_text function in the rvest package.
Images
Images are fairly easy because they are simply files with specific extensions like svg
, png
, gif
etc. In that sense, if one knows the actual location of the image already, a function like download.files can be used to grab it directly. Other times it may not be known, and so we can use a similar approach as before to grab the file.
base_page = read_html('https://en.wikipedia.org/wiki/Main_Page')
picofday_location = base_page %>% # get the main page
html_nodes('#mp-tfp') %>% # extract the div with 'today's featured picture', i.e. tfp
html_nodes('img') %>% # extract the img
html_attr('src') # grab the source location
With location in hand we can now download the file, and even display it in R. The following requires the grid and jpeg packages.
download.file(url=paste0('https:', picofday_location), destfile='img/picofday.jpg', mode='wb')
picofday = jpeg::readJPEG(source='img/picofday.jpg')
df = data.frame(x=rnorm(1), y=rnorm(1)) # note that any random df will suffice
qplot(data=df, geom='blank') +
annotation_custom(grid::rasterGrob(picofday)) +
theme_void()
Note that an alternative approach that might work on some websites would be to extract all the images and then the one with a relevant naming convention. This doesn’t work for the Wikipedia main page because there is nothing to identify which image is the featured picture by file name alone.
Issues
As mentioned, the ease with which you will be able to scrape a website will depend a lot on how well the page/site is put together. Many are cookie-cutter templates designed with no regard to data availability whatsoever, others are just the result of amateurs that do not do web design for a living, and still others are just poorly designed. In addition, some websites may have security or other server back end things to consider that require much of the content to not be made easily available, perhaps intentionally.
On the other hand, other sites will make it as easy as a URL pointing to a csv file, or an API that is easily maneuverable. The main thing is that you must plan ahead for it to either not be easy to get, and/or to be ready for heavy post-processing even when you get the bulk of what you want. The goal should be to make the process automatic such that only a fundamental site or API change will cause you to have to change your code again.
Called by various names: web harvesting, data scraping (presumptuous), etc.↩
Think about your R code, now remove the spaces, indent as irregularly as possible, do not comment anything, use reserved words for common names, don’t name important parts of your site, reject any known coding style, and use default settings. That would describe about 99% of the typical website design I come across.↩
There is subtle distinction between tags versus elements, but which won’t really matter to us.↩
APIs change so regularly that in some cases it might be easier to scrape as above, especially if the site itself changes relatively rarely and still allows direct access to the desired content.↩
I highly recommend Beautiful Soup also. Until rvest and related packages came along, I preferred using Beautiful Soup to what was available in R.↩
Think graphical models rather than spruce.↩
The openfda site was depicted using the Mozilla firebug extension.↩
Technically this data can be downloaded as a supplied csv.↩


I have been able to work on two projects successfully.
Farm Dean Dairy Limited is a website that enables the farmers to get to know about the dairy breeds:ayshire, fresian, Guernsey among others. They are also updated on the latest news in dairy field.
The farmers are able to login and make their orders on the dairy breeds of their choice.
All this was achieved using: Hypertext-Markup language(HTML), Cascading Stylesheet(CSS)and JavaScript.
Online Travel Agents website gives an interface to the local and international tourist to select on the tourist destinations and suitable accomodations. Bookings are done online and any news updates are at the touch of a button.
Languages used include: Hypertext-Markup language(HTML), Cascading Stylesheet(CSS)and JavaScript
Chebet Eddah is a student at Meru University of Science and Technology, pursuing Bachelors Degree in Computer Science. Worked at Kenya Methodist University as an IT support, and part time freelancing.
I have always been passionate about learning programming for software development and webdesign. My pleasure to master and apply this knowledge in my daily life because there exist a wide gap in technology that need to be filled in my society.


Hello {{ listing.listing_name }},
Here at Picasso, we are out to build a review and
recommendation site that allows students to find and
review the best extracurricular teachers, including
you. Your personal profile is
listed on Picasso to help you meet new
students and build your online
reputation. Please take a look below and
verify your information so that new
students can find and contact you.
If you would like to remove your profile
from Picasso, please fill out this brief
request form.
Let us know what you think of Picasso by
replying to this email.
Cheers,
Picasso Team
|


I am a programmer student
I've been learning programming for about a year now. I really want to be a programmer, so I'm learning programming.
I don't have a computer, I work on my uncle's computer ....
I have some special qualifications.
I work in Photoshop with various banners, business cards, photo editing, wedding cards, passport size pictures, stamp size pictures.
I specialize in the above three softwares. I know the type of medium type. Apart from these, I also listen to reading in madrasas. In the first stage, I am studying Hefzul Vibag in Madrasa, then I am studying in General Line, now I am studying in Class Nine, my role is: 29 out of 100 students, I am studying Science. I hope I will become a web developer when I grow up.
Learn moreThis is a longer card with supporting text below as a natural lead-in to additional content. This content is a little bit longer.
Last updated 3 mins ago
This card has supporting text below as a natural lead-in to additional content.
Last updated 3 mins ago
This is a wider card with supporting text below as a natural lead-in to additional content. This card has even longer content than the first to show that equal height action.
Last updated 3 mins ago
With supporting text below as a natural lead-in to additional content.
Go somewhereWith supporting text below as a natural lead-in to additional content.
Go somewhereI studied very well so I was the number one boy in class 2-5 for three years in a row. Hundreds of attempts to get behind me but to no avail. I was very talented. We were five or six students, we studied very well, we used to study till three in the morning one day, this way our competition was cheated, we studied very carefully.


Good:-
1. preventions:-
As air quality is good , chances of diseases caused by air pollutants is less. so,there must be some common awareness to maintain this AQI.
These are:-
1.use public transport and car pool.
2.Maintain good indoor air quality.
3. Do not smoke indoors.
4.Fix water infiltration problems to prevent mould growth
5.Opt for healthy and environmentally friendly household products and materials
6.Limit the use of wood-burning appliances
2.Health effects:-
1.There will be no as such major disease but some common problems can initiate if it is not looked after like Asthma and other lungs problem.
2.The can be the risk of exposure to unhealthy levels of ground-level ozone.
3.During warmer months, Children, who often play outdoors in warmer
weather, are at risk.
3. WHAT CAN I DO?
1.Every time we drive to school, use our heater or air conditioner, clean our windows, or even style our hair, we make choices that affect air pollution. These steps, as well as many others, are things we all can do to help reduce air pollution.
2.Choose environmentally friendly cleaners.
3.Avoid excessive idling of your automobile.
#Moderate:-
1. Prevention:-
1.Avoid areas where the air is polluted.
2.Contribute to the reduction of outdoor air pollution.
3.Do not smoke indoors.
4.Fix water infiltration problems to prevent mould growth.
5.Opt for healthy and environmentally friendly household products and materials
6.Limit the use of wood-burning appliances.
7.Install a carbon monoxide detector, and avoid using appliances that operate on fuel (gasoline, propane, etc.) indoors
8.Measure the concentration of radon in your home with a dosimeter. You can do it yourself or hire an expert
9.Learn about the possible sources of asbestos in your home and take the appropriate precautions, especially if you are doing renovations
10.Opening your windows regularly is an inexpensive and effective measure. However, you should keep your windows shut when outdoor air quality is poor and during periods of extreme cold.
11.wear mask in highly polluted area by checking AQI from your phone.
2.Health effects:-
1.People who are unusually sensitive to ozone or particle pollution may experience respiratory symptoms.
2.The risk of exposure to unhealthy levels of ground-level ozone is greatest.
during warmer months. Children, who often play outdoors in warmer
weather, are at higher risk.
3. chances of asthma and other lungs diseases .
4. skin burn due to global warming.
5.Sensitive individuals will experience more serious conditions. The hearts and respiratory systems of healthy people may be affected.
3. WHAT CAN I DO?
-Participate in your local utility’s energy conservation programs.
-Limit driving by carpooling, using public transportation, biking and walking.
-Combine errands for fewer trips.
-Keep your automobile well tuned and maintained. Follow the manufacturer’s instructions on routine maintenance, such as changing the oil and filters, and checking tire pressure and wheel alignment.
-Avoid excessive idling of your automobile.
-Use electric or hand-powered lawn care equipment.
-Be careful not to spill gasoline when filling up your car or gasoline powered lawn and garden equipment.
-Run dishwashers and clothes washers only when full.
-Choose environmentally friendly cleaners.
-Use water-based or solvent free paints whenever possible and buy products that say "low VOC".


{% extends "static/templates/base.html" %} {% block title %}Results{% endblock %} {% block content %}
The graph on the right visualises how the memory footprint of your application has grown over time. It displays the size of memory available to your application following each garbage collection event.
If the tenured size continually grows significantly over the lifetime of your application, chances are full GC events will become more common, and your application may eventually run out of memory.
Summary Stats
| {{key}} | {{value}} |
| Timestamp | Collector | System | GC Timestamp | YG Util Pre | YG Util Post | YG Size Post | YG Duration | Tenured Util Pre | Tenured Util Post | Tenured Size Post | Tenured Duration | Perm Util Pre | Perm Util Post | Perm Size Post | Heap Util Pre | Heap Util Post | Heap Size Post | GC Duration | User Time | Sys Time | Real Time |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| {{entry.timestamp}} | {{entry.collector}} | {{entry.system}} | {{entry.gc_timestamp}} | {{entry.yg_util_pre}} | {{entry.yg_util_post}} | {{entry.yg_size_post}} | {{entry.yg_pause_time}} | {{entry.tenured_util_pre}} | {{entry.tenured_util_post}} | {{entry.tenured_size_post}} | {{entry.tenured_pause_time}} | {{entry.perm_util_pre}} | {{entry.perm_util_post}} | {{entry.perm_size_post}} | {{entry.heap_util_pre}} | {{entry.heap_util_post}} | {{entry.heap_size_post}} | {{entry.pause_time}} | {{entry.user_time}} | {{entry.sys_time}} | {{entry.real_time}} |
Processed {{ gc_results|length }} entries in {{duration}} secs {% endblock %}


So here is a useful tidbit I came across while setting this whole thing up. I suppose it is somewhat old news to some, however I am new to the github pages game and haven't looked at jekyll since 2010-ish. That last bit is entirely my fault though, as I had left ruby-land at that point and was off in v8-world until recently. When I quit my job and moved back to the united states. Crazy I know.
So after catching up on serial, and getting back in to drawing, I finally got to the point where I could look at code again. It was here as previously mentioned that I got heavily into emacs. Which admittedly was less for the sake of blogging, and more because I had finally committed to get more than a passing familiarity with cl. Which meant spending time in emacs and slime and the more time I spent in emacs the more I found out it can do. This, I am almost certain, is an age-old story of emacs addiction. I wonder if there is a 12 step program for this?


yt ui for angular
trying to write some angular ui
Installation is easy as yt-ui-bootstrap has minimal dependencies - only the AngularJS and Bootstrap's CSS are required.Just checkout dist/ui-yt.min.js
When you are done downloading all the dependencies and project files the only remaining part is to add dependencies on the ui.yt
AngularJS module:
angular.module('myModule', ['ui.yt']);
directive
for ie
directive
trigger once focus
service
busy spin for long time data loading
directive
quick confirm preventing from a slip
service
toast
directive
checkbox list model
service
alert
service
confirm
directive
for dropdown array
ie8+, Chrome, ff
Please check the CONTRIBUTING.md for the contribution guidelines.
nodejs
npm install
gulp
to watch changes, which will trigger tasksgulp b
to just buildsrc
diretory.test
template
folder.spec.js
.html
we're using jasmine and sinon for testing.


SERVICES
Before we sign a contract or write a line of code our team will spend the neccessary
time given to understand your product vision.
The outcome of this scoping seession
is a proposal or high-level statement of work.
Based on your requirements, we can provide
a complete offering or just the pieces you need.