# -*- coding: utf-8 -*-
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
import six
import time
from airflow.exceptions import AirflowException
from airflow.contrib.hooks.databricks_hook import DatabricksHook
from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults
[docs]XCOM_RUN_ID_KEY = 'run_id'
[docs]XCOM_RUN_PAGE_URL_KEY = 'run_page_url'
[docs]def _deep_string_coerce(content, json_path='json'):
"""
Coerces content or all values of content if it is a dict to a string. The
function will throw if content contains non-string or non-numeric types.
The reason why we have this function is because the ``self.json`` field must be a
dict with only string values. This is because ``render_template`` will fail
for numerical values.
"""
c = _deep_string_coerce
if isinstance(content, six.string_types):
return content
elif isinstance(content, six.integer_types + (float,)):
# Databricks can tolerate either numeric or string types in the API backend.
return str(content)
elif isinstance(content, (list, tuple)):
return [c(e, '{0}[{1}]'.format(json_path, i)) for i, e in enumerate(content)]
elif isinstance(content, dict):
return {k: c(v, '{0}[{1}]'.format(json_path, k))
for k, v in list(content.items())}
else:
param_type = type(content)
msg = 'Type {0} used for parameter {1} is not a number or a string' \
.format(param_type, json_path)
raise AirflowException(msg)
[docs]def _handle_databricks_operator_execution(operator, hook, log, context):
"""
Handles the Airflow + Databricks lifecycle logic for a Databricks operator
:param operator: Databricks operator being handled
:param context: Airflow context
"""
if operator.do_xcom_push:
context['ti'].xcom_push(key=XCOM_RUN_ID_KEY, value=operator.run_id)
log.info('Run submitted with run_id: %s', operator.run_id)
run_page_url = hook.get_run_page_url(operator.run_id)
if operator.do_xcom_push:
context['ti'].xcom_push(key=XCOM_RUN_PAGE_URL_KEY, value=run_page_url)
log.info('View run status, Spark UI, and logs at %s', run_page_url)
while True:
run_state = hook.get_run_state(operator.run_id)
if run_state.is_terminal:
if run_state.is_successful:
log.info('%s completed successfully.', operator.task_id)
log.info('View run status, Spark UI, and logs at %s', run_page_url)
return
else:
error_message = '{t} failed with terminal state: {s}'.format(
t=operator.task_id,
s=run_state)
raise AirflowException(error_message)
else:
log.info('%s in run state: %s', operator.task_id, run_state)
log.info('View run status, Spark UI, and logs at %s', run_page_url)
log.info('Sleeping for %s seconds.', operator.polling_period_seconds)
time.sleep(operator.polling_period_seconds)
[docs]class DatabricksSubmitRunOperator(BaseOperator):
"""
Submits a Spark job run to Databricks using the
`api/2.0/jobs/runs/submit
<https://docs.databricks.com/api/latest/jobs.html#runs-submit>`_
API endpoint.
There are two ways to instantiate this operator.
In the first way, you can take the JSON payload that you typically use
to call the ``api/2.0/jobs/runs/submit`` endpoint and pass it directly
to our ``DatabricksSubmitRunOperator`` through the ``json`` parameter.
For example ::
json = {
'new_cluster': {
'spark_version': '2.1.0-db3-scala2.11',
'num_workers': 2
},
'notebook_task': {
'notebook_path'<EMAIL_ADDRESS>},
}
notebook_run = DatabricksSubmitRunOperator(task_id='notebook_run', json=json)
Another way to accomplish the same thing is to use the named parameters
of the ``DatabricksSubmitRunOperator`` directly. Note that there is exactly
one named parameter for each top level parameter in the ``runs/submit``
endpoint. In this method, your code would look like this: ::
new_cluster = {
'spark_version': '2.1.0-db3-scala2.11',
'num_workers': 2
}
notebook_task = {
'notebook_path'<EMAIL_ADDRESS>}
notebook_run = DatabricksSubmitRunOperator(
task_id='notebook_run',
new_cluster=new_cluster,
notebook_task=notebook_task)
In the case where both the json parameter **AND** the named parameters
are provided, they will be merged together. If there are conflicts during the merge,
the named parameters will take precedence and override the top level ``json`` keys.
Currently the named parameters that ``DatabricksSubmitRunOperator`` supports are
- ``spark_jar_task``
- ``notebook_task``
- ``new_cluster``
- ``existing_cluster_id``
- ``libraries``
- ``run_name``
- ``timeout_seconds``
:param json: A JSON object containing API parameters which will be passed
directly to the ``api/2.0/jobs/runs/submit`` endpoint. The other named parameters
(i.e. ``spark_jar_task``, ``notebook_task``..) to this operator will
be merged with this json dictionary if they are provided.
If there are conflicts during the merge, the named parameters will
take precedence and override the top level json keys. (templated)
.. seealso::
For more information about templating see :ref:`jinja-templating`.
https://docs.databricks.com/api/latest/jobs.html#runs-submit
:type json: dict
:param spark_jar_task: The main class and parameters for the JAR task. Note that
the actual JAR is specified in the ``libraries``.
*EITHER* ``spark_jar_task`` *OR* ``notebook_task`` should be specified.
This field will be templated.
.. seealso::
https://docs.databricks.com/api/latest/jobs.html#jobssparkjartask
:type spark_jar_task: dict
:param notebook_task: The notebook path and parameters for the notebook task.
*EITHER* ``spark_jar_task`` *OR* ``notebook_task`` should be specified.
This field will be templated.
.. seealso::
https://docs.databricks.com/api/latest/jobs.html#jobsnotebooktask
:type notebook_task: dict
:param new_cluster: Specs for a new cluster on which this task will be run.
*EITHER* ``new_cluster`` *OR* ``existing_cluster_id`` should be specified.
This field will be templated.
.. seealso::
https://docs.databricks.com/api/latest/jobs.html#jobsclusterspecnewcluster
:type new_cluster: dict
:param existing_cluster_id: ID for existing cluster on which to run this task.
*EITHER* ``new_cluster`` *OR* ``existing_cluster_id`` should be specified.
This field will be templated.
:type existing_cluster_id: str
:param libraries: Libraries which this run will use.
This field will be templated.
.. seealso::
https://docs.databricks.com/api/latest/libraries.html#managedlibrarieslibrary
:type libraries: list of dicts
:param run_name: The run name used for this task.
By default this will be set to the Airflow ``task_id``. This ``task_id`` is a
required parameter of the superclass ``BaseOperator``.
This field will be templated.
:type run_name: str
:param timeout_seconds: The timeout for this run. By default a value of 0 is used
which means to have no timeout.
This field will be templated.
:type timeout_seconds: int32
:param databricks_conn_id: The name of the Airflow connection to use.
By default and in the common case this will be ``databricks_default``. To use
token based authentication, provide the key ``token`` in the extra field for the
connection.
:type databricks_conn_id: str
:param polling_period_seconds: Controls the rate which we poll for the result of
this run. By default the operator will poll every 30 seconds.
:type polling_period_seconds: int
:param databricks_retry_limit: Amount of times retry if the Databricks backend is
unreachable. Its value must be greater than or equal to 1.
:type databricks_retry_limit: int
:param databricks_retry_delay: Number of seconds to wait between retries (it
might be a floating point number).
:type databricks_retry_delay: float
:param do_xcom_push: Whether we should push run_id and run_page_url to xcom.
:type do_xcom_push: bool
"""
# Used in airflow.models.BaseOperator
[docs] template_fields = ('json',)
# Databricks brand color (blue) under white text
@apply_defaults
def __init__(
self,
json=None,
spark_jar_task=None,
notebook_task=None,
new_cluster=None,
existing_cluster_id=None,
libraries=None,
run_name=None,
timeout_seconds=None,
databricks_conn_id='databricks_default',
polling_period_seconds=30,
databricks_retry_limit=3,
databricks_retry_delay=1,
do_xcom_push=False,
**kwargs):
"""
Creates a new ``DatabricksSubmitRunOperator``.
"""
super(DatabricksSubmitRunOperator, self).__init__(**kwargs)
self.json = json or {}
self.databricks_conn_id = databricks_conn_id
self.polling_period_seconds = polling_period_seconds
self.databricks_retry_limit = databricks_retry_limit
self.databricks_retry_delay = databricks_retry_delay
if spark_jar_task is not None:
self.json['spark_jar_task'] = spark_jar_task
if notebook_task is not None:
self.json['notebook_task'] = notebook_task
if new_cluster is not None:
self.json['new_cluster'] = new_cluster
if existing_cluster_id is not None:
self.json['existing_cluster_id'] = existing_cluster_id
if libraries is not None:
self.json['libraries'] = libraries
if run_name is not None:
self.json['run_name'] = run_name
if timeout_seconds is not None:
self.json['timeout_seconds'] = timeout_seconds
if 'run_name' not in self.json:
self.json['run_name'] = run_name or kwargs['task_id']
self.json = _deep_string_coerce(self.json)
# This variable will be used in case our task gets killed.
self.run_id = None
self.do_xcom_push = do_xcom_push
[docs] def get_hook(self):
return DatabricksHook(
self.databricks_conn_id,
retry_limit=self.databricks_retry_limit,
retry_delay=self.databricks_retry_delay)
[docs] def execute(self, context):
hook = self.get_hook()
self.run_id = hook.submit_run(self.json)
_handle_databricks_operator_execution(self, hook, self.log, context)
[docs] def on_kill(self):
hook = self.get_hook()
hook.cancel_run(self.run_id)
self.log.info(
'Task: %s with run_id: %s was requested to be cancelled.',
self.task_id, self.run_id
)
[docs]class DatabricksRunNowOperator(BaseOperator):
"""
Runs an existing Spark job run to Databricks using the
`api/2.0/jobs/run-now
<https://docs.databricks.com/api/latest/jobs.html#run-now>`_
API endpoint.
There are two ways to instantiate this operator.
In the first way, you can take the JSON payload that you typically use
to call the ``api/2.0/jobs/run-now`` endpoint and pass it directly
to our ``DatabricksRunNowOperator`` through the ``json`` parameter.
For example ::
json = {
"job_id": 42,
"notebook_params": {
"dry-run": "true",
"oldest-time-to-consider": "1457570074236"
}
}
notebook_run = DatabricksRunNowOperator(task_id='notebook_run', json=json)
Another way to accomplish the same thing is to use the named parameters
of the ``DatabricksRunNowOperator`` directly. Note that there is exactly
one named parameter for each top level parameter in the ``run-now``
endpoint. In this method, your code would look like this: ::
job_id=42
notebook_params = {
"dry-run": "true",
"oldest-time-to-consider": "1457570074236"
}
python_params = ["douglas adams", "42"]
spark_submit_params = ["--class", "org.apache.spark.examples.SparkPi"]
notebook_run = DatabricksRunNowOperator(
job_id=job_id,
notebook_params=notebook_params,
python_params=python_params,
spark_submit_params=spark_submit_params
)
In the case where both the json parameter **AND** the named parameters
are provided, they will be merged together. If there are conflicts during the merge,
the named parameters will take precedence and override the top level ``json`` keys.
Currently the named parameters that ``DatabricksRunNowOperator`` supports are
- ``job_id``
- ``json``
- ``notebook_params``
- ``python_params``
- ``spark_submit_params``
:param job_id: the job_id of the existing Databricks job.
This field will be templated.
.. seealso::
https://docs.databricks.com/api/latest/jobs.html#run-now
:type job_id: str
:param json: A JSON object containing API parameters which will be passed
directly to the ``api/2.0/jobs/run-now`` endpoint. The other named parameters
(i.e. ``notebook_params``, ``spark_submit_params``..) to this operator will
be merged with this json dictionary if they are provided.
If there are conflicts during the merge, the named parameters will
take precedence and override the top level json keys. (templated)
.. seealso::
For more information about templating see :ref:`jinja-templating`.
https://docs.databricks.com/api/latest/jobs.html#run-now
:type json: dict
:param notebook_params: A dict from keys to values for jobs with notebook task,
e.g. "notebook_params": {"name": "john doe", "age": "35"}.
The map is passed to the notebook and will be accessible through the
dbutils.widgets.get function. See Widgets for more information.
If not specified upon run-now, the triggered run will use the
job’s base parameters. notebook_params cannot be
specified in conjunction with jar_params. The json representation
of this field (i.e. {"notebook_params":{"name":"john doe","age":"35"}})
cannot exceed 10,000 bytes.
This field will be templated.
.. seealso::
https://docs.databricks.com/user-guide/notebooks/widgets.html
:type notebook_params: dict
:param python_params: A list of parameters for jobs with python tasks,
e.g. "python_params": ["john doe", "35"].
The parameters will be passed to python file as command line parameters.
If specified upon run-now, it would overwrite the parameters specified in
job setting.
The json representation of this field (i.e. {"python_params":["john doe","35"]})
cannot exceed 10,000 bytes.
This field will be templated.
.. seealso::
https://docs.databricks.com/api/latest/jobs.html#run-now
:type python_params: list[str]
:param spark_submit_params: A list of parameters for jobs with spark submit task,
e.g. "spark_submit_params": ["--class", "org.apache.spark.examples.SparkPi"].
The parameters will be passed to spark-submit script as command line parameters.
If specified upon run-now, it would overwrite the parameters specified
in job setting.
The json representation of this field cannot exceed 10,000 bytes.
This field will be templated.
.. seealso::
https://docs.databricks.com/api/latest/jobs.html#run-now
:type spark_submit_params: list[str]
:param timeout_seconds: The timeout for this run. By default a value of 0 is used
which means to have no timeout.
This field will be templated.
:type timeout_seconds: int32
:param databricks_conn_id: The name of the Airflow connection to use.
By default and in the common case this will be ``databricks_default``. To use
token based authentication, provide the key ``token`` in the extra field for the
connection.
:type databricks_conn_id: str
:param polling_period_seconds: Controls the rate which we poll for the result of
this run. By default the operator will poll every 30 seconds.
:type polling_period_seconds: int
:param databricks_retry_limit: Amount of times retry if the Databricks backend is
unreachable. Its value must be greater than or equal to 1.
:type databricks_retry_limit: int
:param do_xcom_push: Whether we should push run_id and run_page_url to xcom.
:type do_xcom_push: bool
"""
# Used in airflow.models.BaseOperator
[docs] template_fields = ('json',)
# Databricks brand color (blue) under white text
@apply_defaults
def __init__(
self,
job_id,
json=None,
notebook_params=None,
python_params=None,
spark_submit_params=None,
databricks_conn_id='databricks_default',
polling_period_seconds=30,
databricks_retry_limit=3,
databricks_retry_delay=1,
do_xcom_push=False,
**kwargs):
"""
Creates a new ``DatabricksRunNowOperator``.
"""
super(DatabricksRunNowOperator, self).__init__(**kwargs)
self.json = json or {}
self.databricks_conn_id = databricks_conn_id
self.polling_period_seconds = polling_period_seconds
self.databricks_retry_limit = databricks_retry_limit
self.databricks_retry_delay = databricks_retry_delay
if job_id is not None:
self.json['job_id'] = job_id
if notebook_params is not None:
self.json['notebook_params'] = notebook_params
if python_params is not None:
self.json['python_params'] = python_params
if spark_submit_params is not None:
self.json['spark_submit_params'] = spark_submit_params
self.json = _deep_string_coerce(self.json)
# This variable will be used in case our task gets killed.
self.run_id = None
self.do_xcom_push = do_xcom_push
[docs] def get_hook(self):
return DatabricksHook(
self.databricks_conn_id,
retry_limit=self.databricks_retry_limit,
retry_delay=self.databricks_retry_delay)
[docs] def execute(self, context):
hook = self.get_hook()
self.run_id = hook.run_now(self.json)
_handle_databricks_operator_execution(self, hook, self.log, context)
[docs] def on_kill(self):
hook = self.get_hook()
hook.cancel_run(self.run_id)
self.log.info(
'Task: %s with run_id: %s was requested to be cancelled.',
self.task_id, self.run_id
)


Contributing to the code base
Code standards
Writing good code is not just about what you write. It is also about how you write it. During Continuous Integration testing, several tools will be run to check your code for stylistic errors. Generating any warnings will cause the test to fail. Thus, good style is a requirement for submitting code to pandas.
There is a tool in pandas to help contributors verify their changes before contributing them to the project:
./ci/code_checks.sh
The script verifies the linting of code files, it looks for common
mistake patterns (like missing spaces around sphinx directives that make
the documentation not being rendered properly) and it also validates the
doctests. It is possible to run the checks independently by using the
parameters lint
, patterns
and doctests
(e.g.
./ci/code_checks.sh lint
).
In addition, because a lot of people use our library, it is important that we do not make sudden changes to the code that could have the potential to break a lot of user code as a result, that is, we need it to be as backwards compatible as possible to avoid mass breakages.
Additional standards are outlined on the code style wiki page.
Optional dependencies
Optional dependencies (e.g. matplotlib) should be imported with the
private helper pandas.compat._optional.import_optional_dependency
.
This ensures a consistent error message when the dependency is not met.
All methods using an optional dependency should include a test asserting
that an ImportError
is raised when the optional dependency is not
found. This test should be skipped if the library is present.
All optional dependencies should be documented and the
minimum required version should be set in the
pandas.compat._optional.VERSIONS
dict.
C (cpplint)
pandas uses the
Google standard.
Google provides an open source style checker called cpplint
, but we
use a fork of it that can be found
here. Here are some of the more
common cpplint
issues:
- we restrict line-length to 80 characters to promote readability
- every header file must include a header guard to avoid name collisions if re-included
Continuous Integration will run the cpplint tool and report any stylistic errors in your code. Therefore, it is helpful before submitting code to run the check yourself:
cpplint --extensions=c,h --headers=h --filter=-readability/casting,-runtime/int,-build/include_subdir modified-c-file
You can also run this command on an entire directory if necessary:
cpplint --extensions=c,h --headers=h --filter=-readability/casting,-runtime/int,-build/include_subdir --recursive modified-c-directory
To make your commits compliant with this standard, you can install the ClangFormat tool, which can be downloaded here. To configure, in your home directory, run the following command:
clang-format style=google -dump-config > .clang-format
Then modify the file to ensure that any indentation width parameters are at least four. Once configured, you can run the tool as follows:
clang-format modified-c-file
This will output what your file will look like if the changes are made, and to apply them, run the following command:
clang-format -i modified-c-file
To run the tool on an entire directory, you can run the following analogous commands:
clang-format modified-c-directory/*.c modified-c-directory/*.h
clang-format -i modified-c-directory/*.c modified-c-directory/*.h
Do note that this tool is best-effort, meaning that it will try to
correct as many errors as possible, but it may not correct all of
them. Thus, it is recommended that you run cpplint
to double check and
make any other style fixes manually.
Python (PEP8 / black)
pandas follows the PEP8 standard and uses Black and Flake8 to ensure a consistent code format throughout the project.
Continuous Integration will run those tools and report any stylistic errors in your code. Therefore, it is helpful before submitting code to run the check yourself:
black pandas
git diff upstream/master -u -- "*.py" | flake8 --diff
to auto-format your code. Additionally, many editors have plugins that
will apply black
as you edit files.
Optionally, you may wish to setup pre-commit
hooks to automatically run black
and
flake8
when you make a git commit. This can be done by installing
pre-commit
:
pip install pre-commit
and then running:
pre-commit install
from the root of the pandas repository.  You can skip these checks with
git commit --no-verify
.
This command will catch any stylistic errors in your changes specifically, but be beware it may not catch all of them. For example, if you delete the only usage of an imported function, it is stylistically incorrect to import an unused function. However, style-checking the diff will not catch this because the actual import is not part of the diff. Thus, for completeness, you should run this command, though it will take longer:
git diff upstream/master --name-only -- "*.py" | xargs -r flake8
Note that on OSX, the -r
flag is not available, so you have to omit it
and run this slightly modified command:
git diff upstream/master --name-only -- "*.py" | xargs flake8
Windows does not support the xargs
command (unless installed for
example via the MinGW toolchain), but one can
imitate the behaviour as follows:
for /f %i in ('git diff upstream/master --name-only -- "*.py"') do flake8 %i
This will get all the files being changed by the PR (and ending with
.py
), and run flake8
on them, one after the other.
Import formatting
pandas uses isort to standardise import formatting across the codebase.
A guide to import layout as per pep8 can be found here.
A summary of our current import sections ( in order ):
- Future
- Python Standard Library
- Third Party
pandas._libs
,pandas.compat
,pandas.util._*
,pandas.errors
(largely not dependent onpandas.core
)pandas.core.dtypes
(largely not dependent on the rest ofpandas.core
)- Rest of
pandas.core.*
- Non-core
pandas.io
,pandas.plotting
,pandas.tseries
- Local application/library specific imports
Imports are alphabetically sorted within these sections.
As part of Continuous Integration checks we run:
isort --recursive --check-only pandas
to check that imports are correctly formatted as per the
setup.cfg
.
If you see output like the below in Continuous Integration checks:
Check import format using isort
ERROR: /home/travis/build/pandas-dev/pandas/pandas/io/pytables.py Imports are incorrectly sorted
Check import format using isort DONE
The command "ci/code_checks.sh" exited with 1
You should run:
isort pandas/io/pytables.py
to automatically format imports correctly. This will modify your local copy of the files.
The --recursive
flag can be passed to sort all files in a
directory.
You can then verify the changes look ok, then git commit and push.
Backwards compatibility
Please try to maintain backward compatibility. pandas has lots of users with lots of existing code, so don\'t break it if at all possible. If you think breakage is required, clearly state why as part of the pull request. Also, be careful when changing method signatures and add deprecation warnings where needed. Also, add the deprecated sphinx directive to the deprecated functions or methods.
If a function with the same arguments as the one being deprecated exist,
you can use the pandas.util._decorators.deprecate
:
from pandas.util._decorators import deprecate
deprecate('old_func', 'new_func', '0.21.0')
Otherwise, you need to do it manually:
import warnings
def old_func():
"""Summary of the function.
.. deprecated:: 0.21.0
Use new_func instead.
"""
warnings.warn('Use new_func instead.', FutureWarning, stacklevel=2)
new_func()
def new_func():
pass
You'll also need to
- Write a new test that asserts a warning is issued when calling with the deprecated argument
- Update all of pandas existing tests and code to use the new argument
See contributing warnings for more.
Testing with continuous integration
The pandas test suite will run automatically on Travis-CI and Azure Pipelines continuous integration services, once your pull request is submitted. However, if you wish to run the test suite on a branch prior to submitting the pull request, then the continuous integration services need to be hooked to your GitHub repository. Instructions are here for Travis-CI and Azure Pipelines.
A pull-request will be considered for merging when you have an all \'green\' build. If any tests are failing, then you will get a red \'X\', where you can click through to see the individual failed tests. This is an example of a green build.
Each time you push to your fork, a new run of the tests will be triggered on the CI. You can enable the auto-cancel feature, which removes any non-currently-running tests for that same pull-request, for Travis-CI here.
Test-driven development/code writing
pandas is serious about testing and strongly encourages contributors to embrace test-driven development (TDD). This development process "relies on the repetition of a very short development cycle: first the developer writes an (initially failing) automated test case that defines a desired improvement or new function, then produces the minimum amount of code to pass that test." So, before actually writing any code, you should write your tests. Often the test can be taken from the original GitHub issue. However, it is always worth considering additional use cases and writing corresponding tests.
Adding tests is one of the most common requests after code is pushed to pandas. Therefore, it is worth getting in the habit of writing tests ahead of time so this is never an issue.
Like many packages, pandas uses pytest and the convenient extensions in numpy.testing.
The earliest supported pytest version is 4.0.2.
Writing tests
All tests should go into the tests
subdirectory of the specific
package. This folder contains many current examples of tests, and we
suggest looking to these for inspiration. If your test requires working
with files or network connectivity, there is more information on the
testing page
of the wiki.
The pandas.util.testing
module has many special assert
functions
that make it easier to make statements about whether Series or DataFrame
objects are equivalent. The easiest way to verify that your code is
correct is to explicitly construct the result you expect, then compare
the actual result to the expected correct result:
def test_pivot(self):
data = {
'index' : ['A', 'B', 'C', 'C', 'B', 'A'],
'columns' : ['One', 'One', 'One', 'Two', 'Two', 'Two'],
'values' : [1., 2., 3., 3., 2., 1.]
}
frame = DataFrame(data)
pivoted = frame.pivot(index='index', columns='columns', values='values')
expected = DataFrame({
'One' : {'A' : 1., 'B' : 2., 'C' : 3.},
'Two' : {'A' : 1., 'B' : 2., 'C' : 3.}
})
assert_frame_equal(pivoted, expected)
Transitioning to pytest
pandas existing test structure is mostly classed based, meaning that you will typically find tests wrapped in a class.
class TestReallyCoolFeature:
pass
Going forward, we are moving to a more functional style using the pytest framework, which offers a richer testing framework that will facilitate testing and developing. Thus, instead of writing test classes, we will write test functions like this:
def test_really_cool_feature():
pass
Using pytest
Here is an example of a self-contained set of tests that illustrate multiple features that we like to use.
- functional style: tests are like
test_*
and only take arguments that are either fixtures or parameters pytest.mark
can be used to set metadata on test functions, e.g.skip
orxfail
.- using
parametrize
: allow testing of multiple cases - to set a mark on a parameter,
pytest.param(..., marks=...)
syntax should be used fixture
, code for object construction, on a per-test basis- using bare
assert
for scalars and truth-testing tm.assert_series_equal
(and its counter parttm.assert_frame_equal
), for pandas object comparisons.- the typical pattern of constructing an
expected
and comparing versus theresult
We would name this file test_cool_feature.py
and put in an appropriate
place in the pandas/tests/
structure.
import pytest
import numpy as np
import pandas as pd
@pytest.mark.parametrize('dtype', ['int8', 'int16', 'int32', 'int64'])
def test_dtypes(dtype):
assert str(np.dtype(dtype)) == dtype
@pytest.mark.parametrize(
'dtype', ['float32', pytest.param('int16', marks=pytest.mark.skip),
pytest.param('int32', marks=pytest.mark.xfail(
reason='to show how it works'))])
def test_mark(dtype):
assert str(np.dtype(dtype)) == 'float32'
@pytest.fixture
def series():
return pd.Series([1, 2, 3])
@pytest.fixture(params=['int8', 'int16', 'int32', 'int64'])
def dtype(request):
return request.param
def test_series(series, dtype):
result = series.astype(dtype)
assert result.dtype == dtype
expected = pd.Series([1, 2, 3], dtype=dtype)
tm.assert_series_equal(result, expected)
A test run of this yields
((pandas) bash-3.2$ pytest test_cool_feature.py -v
=========================== test session starts ===========================
platform darwin -- Python 3.6.2, pytest-3.6.0, py-1.4.31, pluggy-0.4.0
collected 11 items
tester.py::test_dtypes[int8] PASSED
tester.py::test_dtypes[int16] PASSED
tester.py::test_dtypes[int32] PASSED
tester.py::test_dtypes[int64] PASSED
tester.py::test_mark[float32] PASSED
tester.py::test_mark[int16] SKIPPED
tester.py::test_mark[int32] xfail
tester.py::test_series[int8] PASSED
tester.py::test_series[int16] PASSED
tester.py::test_series[int32] PASSED
tester.py::test_series[int64] PASSED
Tests that we have parametrized
are now accessible via the test name,
for example we could run these with -k int8
to sub-select only those
tests which match int8
.
((pandas) bash-3.2$ pytest test_cool_feature.py -v -k int8
=========================== test session starts ===========================
platform darwin -- Python 3.6.2, pytest-3.6.0, py-1.4.31, pluggy-0.4.0
collected 11 items
test_cool_feature.py::test_dtypes[int8] PASSED
test_cool_feature.py::test_series[int8] PASSED
Using hypothesis
Hypothesis is a library for property-based testing. Instead of explicitly parametrizing a test, you can describe all valid inputs and let Hypothesis try to find a failing input. Even better, no matter how many random examples it tries, Hypothesis always reports a single minimal counterexample to your assertions - often an example that you would never have thought to test.
See Getting Started with Hypothesis for more of an introduction, then refer to the Hypothesis documentation. for details.
import json
from hypothesis import given, strategies as st
any_json_value = st.deferred(lambda: st.one_of(
st.none(), st.booleans(), st.floats(allow_nan=False), st.text(),
st.lists(any_json_value), st.dictionaries(st.text(), any_json_value)
))
@given(value=any_json_value)
def test_json_roundtrip(value):
result = json.loads(json.dumps(value))
assert value == result
This test shows off several useful features of Hypothesis, as well as demonstrating a good use-case: checking properties that should hold over a large or complicated domain of inputs.
To keep the Pandas test suite running quickly, parametrized tests are preferred if the inputs or logic are simple, with Hypothesis tests reserved for cases with complex logic or where there are too many combinations of options or subtle interactions to test (or think of!) all of them.
Testing warnings
By default, one of pandas CI workers will fail if any unhandled warnings are emitted.
If your change involves checking that a warning is actually emitted, use
tm.assert_produces_warning(ExpectedWarning)
.
import pandas.util.testing as tm
df = pd.DataFrame()
with tm.assert_produces_warning(FutureWarning):
df.some_operation()
We prefer this to the pytest.warns
context manager because ours checks
that the warning\'s stacklevel is set correctly. The stacklevel is what
ensure the user\'s file name and line number is printed in the
warning, rather than something internal to pandas. It represents the
number of function calls from user code (e.g. df.some_operation()
) to
the function that actually emits the warning. Our linter will fail the
build if you use pytest.warns
in a test.
If you have a test that would emit a warning, but you aren\'t actually
testing the warning itself (say because it\'s going to be removed in the
future, or because we\'re matching a 3rd-party library\'s behavior),
then use pytest.mark.filterwarnings
to ignore the error.
@pytest.mark.filterwarnings("ignore:msg:category")
def test_thing(self):
...
If the test generates a warning of class category
whose message starts
with msg
, the warning will be ignored and the test will pass.
If you need finer-grained control, you can use Python\'s usual warnings module to control whether a warning is ignored / raised at different places within a single test.
with warnings.catch_warnings():
warnings.simplefilter("ignore", FutureWarning)
# Or use warnings.filterwarnings(...)
Alternatively, consider breaking up the unit test.
Running the test suite
The tests can then be run directly inside your Git clone (without having to install pandas) by typing:
pytest pandas
The tests suite is exhaustive and takes around 20 minutes to run. Often it is worth running only a subset of tests first around your changes before running the entire suite.
The easiest way to do this is with:
pytest pandas/path/to/test.py -k regex_matching_test_name
Or with one of the following constructs:
pytest pandas/tests/[test-module].py
pytest pandas/tests/[test-module].py::[TestClass]
pytest pandas/tests/[test-module].py::[TestClass]::[test_method]
Using pytest-xdist, one can speed up local testing on multicore machines. To use this feature, you will need to install pytest-xdist via:
pip install pytest-xdist
Two scripts are provided to assist with this. These scripts distribute testing across 4 threads.
On Unix variants, one can type:
test_fast.sh
On Windows, one can type:
test_fast.bat
This can significantly reduce the time it takes to locally run tests before submitting a pull request.
For more, see the pytest documentation.
Furthermore one can run
pd.test()
with an imported pandas to run tests similarly.
Running the performance test suite
Performance matters and it is worth considering whether your code has
introduced performance regressions. pandas is in the process of
migrating to asv benchmarks to
enable easy monitoring of the performance of critical pandas
operations. These benchmarks are all found in the pandas/asv_bench
directory. asv supports both python2 and python3.
To use all features of asv, you will need either conda
or
virtualenv
. For more details please check the
asv installation webpage.
To install asv:
pip install git+https://github.com/spacetelescope/asv
If you need to run a benchmark, change your directory to asv_bench/
and run:
asv continuous -f 1.1 upstream/master HEAD
You can replace HEAD
with the name of the branch you are working on,
and report benchmarks that changed by more than 10%. The command uses
conda
by default for creating the benchmark environments. If you want
to use virtualenv instead, write:
asv continuous -f 1.1 -E virtualenv upstream/master HEAD
The -E virtualenv
option should be added to all asv
commands that
run benchmarks. The default value is defined in asv.conf.json
.
Running the full test suite can take up to one hour and use up to 3GB of
RAM. Usually it is sufficient to paste only a subset of the results into
the pull request to show that the committed changes do not cause
unexpected performance regressions. You can run specific benchmarks
using the -b
flag, which takes a regular expression. For example, this
will only run tests from a pandas/asv_bench/benchmarks/groupby.py
file:
asv continuous -f 1.1 upstream/master HEAD -b ^groupby
If you want to only run a specific group of tests from a file, you can
do it using .
as a separator. For example:
asv continuous -f 1.1 upstream/master HEAD -b groupby.GroupByMethods
will only run the GroupByMethods
benchmark defined in groupby.py
.
You can also run the benchmark suite using the version of pandas
already installed in your current Python environment. This can be useful
if you do not have virtualenv or conda, or are using the
setup.py develop
approach discussed above; for the in-place build you
need to set PYTHONPATH
, e.g.
PYTHONPATH="$PWD/.." asv [remaining arguments]
. You can run benchmarks
using an existing Python environment by:
asv run -e -E existing
or, to use a specific Python interpreter,:
asv run -e -E existing:python3.5
This will display stderr from the benchmarks, and use your local
python
that comes from your $PATH
.
Information on how to write a benchmark and how to use asv can be found in the asv documentation.
Documenting your code
Changes should be reflected in the release notes located in
doc/source/whatsnew/vx.y.z.rst
. This file contains an ongoing change
log for each release. Add an entry to this file to document your fix,
enhancement or (unavoidable) breaking change. Make sure to include the
GitHub issue number when adding your entry (using :issue:`1234
where 1234
is the issue/pull request
number).
If your code is an enhancement, it is most likely necessary to add usage
examples to the existing documentation. This can be done following the
section regarding documentation above.
Further, to let users know when this feature was added, the
versionadded
directive is used. The sphinx syntax for that is:
This will put the text New in version 0.21.0 wherever you put the sphinx directive. This should also be put in the docstring when adding a new function or method (example) or a new keyword argument (example).


Knowledge Base
Htmlstream community is your first stop for questions and advice about the UI Kit. Welcome to the community!
Getting started
Welcome to Docs UI Kit! We're so glad you're here. Let's get started!
Account
Adjust your profile and preferences to make Docs UI Kit work just for you!
Subscription
Assistance on how and when you might use the subscription product.
Tips, tricks & more
Tips and tools for beginners and experts alike.
Last articles
Popular articles
Popular forum topics
Most Helpful Articles
Start here to find best possible answers from our experts.
Getting started for workspace creators
Nick Brown
1 minute ago
View and download invoices
Mary Gordon
15 minutes ago
Getting started for new members
Tom Pook
2 hour ago
Invite new members to your workspace
Helena Shnider
3 hour ago


--- layout: post title: "JavaScript Snake Game" date: 2016-01-04 12:00:00 -0500 comments: true categories: jekyll update tags: JavaScript ---
This snake game is written in JavaScript after I've read the tutorial in This Link
You are welcome to play with it XP.


{% autoescape off %}
Hi, {{ user.username }}.
We received a request for a password reset. If this was you,
follow the link below to reset your password. If this wasn't you, no action is needed.
{{ protocol }}://{{ domain }}{% url 'password_reset_confirm' uidb64=uid token=token %}
If clicking the link above doesn't work, please copy and paste the URL in a new browser
window instead.
Sincerely,
Persyda's Kitchen
{% endautoescape %}


Welcome to PIGOR’s documentation!¶
Pigor is a lightweight analysis tool for the the polarimeter instrument NEPTUN beam port of the 250kW research reactor hosted at Atominstitut of TU Wien, Austria. For more information visit our homepage.
Old ToDos¶
Note
This list of ToDos should be integrated into the docstrings or into the sprint planning where those ToDos belong.
Here are all ToDos listed. Feel free to contribute and check this project out on Bitbucket.
- self.x_error: not yet implemented
- a lot of commenting
- error of fit
- verbose mode on/off
- getting PIGOR ready for shipment by creating a setup.py
- better display of self.pcov
- separation between pure functions and functions with context (methods)
- auto comment decorator for functions
- use decorators to auto register fit functions with their input argument list
- setuptools in setup.py
- https://click.palletsprojects.com/en/7.x/quickstart/
Project Dependencies¶
Todo
check if dependencies are correct with dependencies.txt
- numpy
- re
- matplotlib
- os.path
- glob
- difflib
- datetime
- pathlib
- scipy
- markdown


{% extends "webclient/_logged_base.html" %}
{% set title="Lab - " + experiment['display_name'] %}
{% import "webclient/_upload.html" as upload %}
{% block styles %}
{{ super() }}
{{ upload.styles() }}
{% endblock %}
{% block body %}
{{ super() }}
{# Includes the base HTML for the angular App for the laboratory, so that it is (or seems) more modular. #}
{% include 'webclient/apps/lab/lab.html' %}
{% endblock %}
{% block scripts %}
{{ super() }}
{{ upload.scripts() }}
{{ upload.code(config["UPLOAD_URL"]) }}
{# TODO: Switch for .min files on release #}
{# Embedded templates #}
{% include 'webclient/apps/lab/expinfo/expinfo.template.html' %}
{% include 'webclient/apps/lab/reserve-status/reserve-status.directive.html' %}
{% include 'webclient/apps/lab/iframe/experiment-iframe.template.html' %}
{# Fake includes so that we can use Jinja2 directives #}
{% endblock %}


www.coastlinemicro.com
Coastline Core Business
•
Founded in 1989
•
Privately held
•
Irvine, CA. Corporate Headquarters
•
Build to Order Servers, Desktops &
Notebooks
•
Component Sales
•
Networking Services
•
National Onsite Warranties
•


Browser events
Execute all handlers and behaviors attached to the matched elements for the given event type.
(HTMLElement)
The target object for events dispatched to the EventTarget object.
(String)
The type is the name of Event object to be dispatched.
(Mixed)
Attach an event handler function for one or more events to the selected elements.
(HTMLElement)
The target object for events dispatched to the EventTarget object.
(String)
The type is the name of Event object to be dispatched.
(Function)
The listener function that processes the event.
This function must accept an Event object as its only parameter and must return nothing.
(Object)
An options object that specifies characteristics about the
event listener.
(Boolean)
Determines whether the listener works in the capture phase or
the target and bubbling phases. If useCapture is set to true, the listener processes the
event only during the capture phase and not in the target or bubbling phase.
If useCapture is false, the listener processes the event only during the target or
bubbling phase.
To listen for the event in all three phases, call addEventListener twice, once
with useCapture set to true, then again with useCapture set to false.
Remove event handler(s) from element(s).
(HTMLElement)
The target object for events dispatched to the EventTarget object.
(String)
The type is the name of Event object to be dispatched.
(Function)
The listener function that processes the event.
This function must accept an Event object as its only parameter and must return nothing.
(Object)
An options object that specifies characteristics about the
event listener.
(Boolean)
Determines whether the listener works in the capture phase or
the target and bubbling phases. If useCapture is set to true, the listener processes the
event only during the capture phase and not in the target or bubbling phase.
If useCapture is false, the listener processes the event only during the target or
bubbling phase.
To listen for the event in all three phases, call addEventListener twice, once
with useCapture set to true, then again with useCapture set to false.
Attach a handler to an event for the elements. The handler is executed at most one per element per event type.
(HTMLElement)
The target object for events dispatched to the EventTarget object.
(String)
The type is the name of Event object to be dispatched.
(Function)
The listener function that processes the event.
This function must accept an Event object as its only parameter and must return nothing.
(Object)
An options object that specifies characteristics about the
event listener.
(Boolean)
Determines whether the listener works in the capture phase or
the target and bubbling phases. If useCapture is set to true, the listener processes the
event only during the capture phase and not in the target or bubbling phase.
If useCapture is false, the listener processes the event only during the target or
bubbling phase.
To listen for the event in all three phases, call addEventListener twice, once
with useCapture set to true, then again with useCapture set to false.
Checks whether the dom-event-dispatcher
object has any listeners registered for a specific
type of event. This allows you to determine where an dom-event-dispatcher
object has altered
handling of an event type in the event flow hierarchy. To determine whether a specific event
type actually triggers an event listener, use willEmit()
.
The difference between hasEvent()
and willEmit()
is that hasEvent()
examines only the
object to which it belongs, whereas willEmit()
examines the entire event flow for the event
specified by the type
parameter.
(HTMLElement)
The target object for events dispatched to the EventTarget object.
(any)
(String)
The type is the name of Event object to be dispatched.
Boolean
:
A value of
true
if a listener of the specified type is registered;
false
otherwise.
Checks whether an event listener is registered with this dom-event-dispatcher
object or
any of its ancestors for the specified event type. This method returns true
if an event
listener is triggered during any phase of the event flow when an event of the specified
type is dispatched to this dom-event-dispatcher
object or any of its descendants.
The difference between the hasEvent()
and the willEmit()
methods is that hasEvent()
examines only the object to which it belongs, whereas the willEmit()
method examines
the entire event flow for the event specified by the type parameter.
(any)
(String)
The type is the name of Event object to be dispatched.
Boolean
:
A value of
true
if a listener of the specified type is registered;
false
otherwise.
Creates an event of the type specified. The returned object should be first
initialized and can then be passed to element.dispatchEvent
.
(String)
The type is the name of Event object to be dispatched.
Possible event types include "UIEvents", "MouseEvents", "MutationEvents",
and "HTMLEvents".
(any)
Browser events polyfill
(any)
(any)
The EventTarget.addEventListener() method adds the specified EventListener-compatible object to the list of event listeners for the specified event type on the EventTarget on which it's called. The event target may be an Element in a document, the Document itself, a Window, or any other object that supports events (such as XMLHttpRequest).
(HTMLElement)
The target object for events dispatched to the EventTarget object.
(String)
The type is the name of Event object to be dispatched.
(Function)
The listener function that processes the event.
This function must accept an Event object as its only parameter and must return nothing.
(Object)
An options object that specifies characteristics about the event
listener.
(Boolean)
Determines whether the listener works in the capture phase or
the target and bubbling phases. If useCapture is set to true, the listener processes the
event only during the capture phase and not in the target or bubbling phase.
If useCapture is false, the listener processes the event only during the target or bubbling
phase.
To listen for the event in all three phases, call addEventListener twice, once
with useCapture set to true, then again with useCapture set to false.
The EventTarget.removeEventListener() method removes from the EventTarget an event listener previously registered with EventTarget.addEventListener(). The event listener to be removed is identified using a combination of the event type, the event listener function itself, and various optional options that may affect the matching process; see Matching event listeners for removal.
(HTMLElement)
The target object for events dispatched to the EventTarget object.
(String)
The type is the name of Event object to be dispatched.
(Function)
The listener function that processes the event.
This function must accept an Event object as its only parameter and must return nothing.
(Object)
An options object that specifies characteristics about the event
listener.
(Boolean)
Determines whether the listener works in the capture phase or
the target and bubbling phases. If useCapture is set to true, the listener processes the
event only during the capture phase and not in the target or bubbling phase.
If useCapture is false, the listener processes the event only during the target or
bubbling phase.
To listen for the event in all three phases, call addEventListener twice, once with useCapture
set to true, then again with useCapture set to false.
Dispatches an Event at the specified EventTarget, invoking the affected EventListeners in the appropriate order. The normal event processing rules (including the capturing and optional bubbling phase) also apply to events dispatched manually with dispatchEvent().
(HTMLElement)
The target object for events dispatched to the EventTarget object.
(String)
The type is the name of Event object to be dispatched.
(Mixed)
Boolean
:
cancelled Indicating whether the event was canceled
by an event handler.


Hi!
Currently I'm part of Thunderbird School of Global Managements pioneer undergrad program, as well as a communication major with ASU's The New College. I'm choosing to pursue a double major because I believe that globalization has created a competitive space that demands the innovativeness that a diverse workforce brings. My goal is to be the glue that ties those people together, because true productivity begins with cohesiveness. My management experience working with medical healthcare teams, as well as my education has given me the ability to create and mediate that collaborative space.
GPA: 3.20
GPA: 3.20
GPA: 3.83 Weighted
Writing | Reading | Dancing
I love doing activities that allow me to express myself in creative ways. I alsolove spending time with my dog Yogi, who I consider my heart and my soul.


Recent Advances in Few-Shot Learning
Few-shot learning is (in my opinion) one of the most interesting and important research areas in ML. It touches at the very core of what ML/DL can’t do today, and is one of the clear indicators of how much work is left. All the approaches to few-shot learning I am aware of use probabilistic modelling. In fact, one of the reasons I’m so sold on Bayesian learning is that it offers avenues to pursue these problems - its not clear to me how one can even approach this not from a probabilistic perspective.
A Discriminative Approach
An interesting paper1 by Rich Turner's group (in collaboration with Bernard Scholkopf's group) proposed using the representations found in the final hidden layer of a deepNet to generalize to unseen classes for classification.
By placing a prior on the weights they can perform posterior inference on the values for a new class, and show impressive results using very simple priors and likelihood functions. This is the only discriminative approach I am familiar with, and I think it shows a lot of promise for future work.
Generative Models for Few-Shot Learning
More common are the use of generative models. These can largely be placed on a spectrum of how structured the models are.
Unstructured Models
On one end of the generative spectrum are completely unstructured models [2, 3]. Here, researchers attempt to learn meaningful representations in latent variables, and then make use of that ‘information’ for new tasks. This has shown good results, even though optimal mechanisms for information sharing are not yet clear. A lot of interesting work is being done here.
Heavily Structured Models
On the other hand are heavily structured models [4, 5, 6]. These works come from a faction of ML researchers who believe we should encode as much knowledge as possible into our models [7]. For example, in Kansky et al.5 show how a heavily engineered model can transfer its capabilities to unseen scenarios in game playing, something that state of the art deep RL agents are catastrophically bad at.
There is a tradeoff here: structured models achieve very impressive generalization, sample efficiency, and few-shot capabilities. It is clear that by encoding knowledge we can achieve significant gains in these aspects. On the other hand, this entails expert knowledge and the ability to encode it in a computationally tractable manner. This is obviously not possible for all domains, and it may not be a scalable approach to improving AI [8]. Conversely, unstructured generative modelling relieves the burden of human engineering, which is a clear goal of ML. However, it is not clear how to guide the latent variables towards useful representations, nor is it clear what is the ‘right’ way to then transfer this knowledge.
One interesting recent paper9 proposes a method for encoding known axes of variation in partially observed variables while allowing latent variables to capture unknown axes of variation. The paper proposes a general approach for variational inference in these models. I believe this approach of finding the middle ground of the spectrum will be very influential in the near future.
Originally a Quora answer
References
-
Bauer, Matthias, et al. Discriminative K-Shot Learning Using Probabilistic Models. 2017 ↩
-
Rezende J., Danilo, et al. One-Shot Generalization in Deep Generative Models. 2016 ↩
-
Edwards, Harrison, and Storkey, Amos. Towards a Neural Statistician. 2017 ↩
-
Salakhutdinov, Ruslan, Tenenbaum, Joshua, Torralba, Antonio. One-Shot Learning with a Hierarchical Nonparametric Bayesian Model. 2012 ↩
-
Kansky, Ken, et al. Schema Networks: Zero-Shot Transfer with a Generative Causal Model of Intuitive Physics. 2017 ↩
-
George, Dileep, et al. A Generative Vision Model that Trains with High Data-Efficiency and Breaks Text-Based CAPTCHAs. 2017 ↩
-
Lake M., Brenden, et al. Building Machines that Learn and Think Like People. 2017 ↩
-
Botvinick, Matthew, et al. Building Machines that Learn and Think for Themselves: Commentary on Lake, Ullman, Tenebaum, and Gershamn. 2017 ↩
-
Siddarth, N., et al. Learning Disentangled Representations with Semi-Supervised Deep Generative Models. 2017 ↩


You must be signed into Google using your umich email address.
Click on an open time to sign up. Add the topic you'd like to get more comfortable with to the description.
To cancel a tutoring session, delete the event from your own Google calendar and let the tutor know via email.


--- layout: post status: publish published: true title: Trivial Properties author: display_name: Tomas Restrepo login: tomasr email<EMAIL_ADDRESS>url: http://winterdom.com/ author_login: tomasr author_email<EMAIL_ADDRESS>author_url: http://winterdom.com/ wordpress_id: 756 wordpress_url: http://winterdom.com/2004/05/trivialproperties date: '2004-05-12 15:54:57 +0000' date_gmt: '2004-05-12 15:54:57 +0000' categories: - ".NET" tags: [] comments: [] ---
One of the features I'm liking more in C++/CLI is Trivial Properties:
#using
using namespace System;
ref class X
{
public:
property int Value;
};
int main()
{
X^ x = gcnew X();
x->Value = 4;
Console::WriteLine("Value is {0}", x->get_Value());
}
Pure syntactic-sugar, but saves a lot of typing...


 I hadn't seen that term before, but I quite like it.
'Latent Typing' is also the subject of Bruce's keynote at PyCon in a few weeks --
http://dc2004reg.pycon.org/dc2004/talks/keynote
in this talk I will look at various issues and arguments surrounding the concept of type, and in particular examine the phenomenon of 'latent typing' (often called 'weak typing'), why the concept is powerful, and how it is expressed in different languages. In the process, I will attempt to clear up many of the issues that have arisen during attempts to describe Python's place in the spectrum of language features
...wish I was going to be there. Hope that he puts the text up on his website.
And does Bruce's novel term provide clarity or confusion?
What is the relationship between an explicit Java interface and a latent type?
What is the relationship between an explicit interface and an implicit interface?
http://www.jot.fm/issues/issue_2002_05/article1# Isaac Gouy


Lorem ipsum dolor sit amet, consectetur adipisicing elit. Minima maxime quam architecto quo inventore harum ex magni, dicta impedit.
Lorem ipsum dolor sit amet, consectetur adipisicing elit. Minima maxime quam architecto quo inventore harum ex magni, dicta impedit.
Lorem ipsum dolor sit amet, consectetur adipisicing elit. Minima maxime quam architecto quo inventore harum ex magni, dicta impedit.
I attended the University of Massachusetts Amherst for my undergraduate degree. I graduated cum laude with general studies honors with a B.Sc. in Mathematics with a concentration in Applied Mathematics and a minor in Computer Science.
After graduating, I worked as and Integration Engineer on the Electronic Data Interchange team at Epic Systems in Verona, WI. While there I worked as a software developer with expertise in orders and results interfaces using HL7 V2 messaging standards, as well as an Interface Coordinator during an Epic instiallation project, and provided tier three techinical support.
I am currently a PhD student at the University of Maryland College Park in the Applied Mathematics & Statistics, and Scientific Computing program.


logtalk_allure_report - Logtalk script for generating Allure reports
logtalk_allure_report -v logtalk_allure_report is a script for generating Allure reports (version 2.21.0 or later). Requires running first the logtalk_tester script using the -f xunit (or -f xunit_net_v2) command-line option.
-v Print script version.
-d The directory containing the tests (i.e. the directory where the logtalk_tester script was run; default is the current directory).
-i Directory where to collect the test results (i.e. the xUnit reports) for generating the report (default is ./allure-results).
-o Directory where the report is to be saved (default is ./allure-report). If the directory already exists and contains a previous report, its contents are deleted before generating the new report. If the directory exists, is not empty, and does not appear to contain a previous report, the script execution is terminated to prevent data loss.
-t Report title (default is "Allure Report"; this option is ignored when using the -p option).
-p Preprocess results but do not generate report. Useful when the actual generatation of the report is delegated to e.g. a Docker service or a Jenkins plug-in.
-- Environment pairs (key1=value1 key2=value2 ...). Used to generate an environment.properties for adding additional information to the generated report.
-h Provide help on script usage.
0 Allure report successfully created.
1 Either the results directory does not exist or the report directory exists but it is not empty and does not contain a previous report.
Run all tests on the current directory and sub-directories using the GNU Prolog backend (saving the test results in the required xUnit format), generate the Allure report, and open the report for browsing: logtalk_tester -p gnu -f xunit
Set the title for the report and information on the backend used: logtalk_allure_report -t "My application tests report" -- Backend='GNU Prolog' Commit=f7a106e
Save the processed test results and the report data to permanent directories so that the report shows tests run trends: logtalk_allure_report -i "$HOME/my_project/allure-results" -o "$HOME/my_project/allure_report" Tested using Allure 2.21.0.
The Allure website at https://docs.qameta.io/allure-report/
The Logtalk website at https://logtalk.org/
The Logtalk User and Reference Manuals at $LOGTALKHOME/manuals/index.html
Paulo Moura<EMAIL_ADDRESS>Copyright (c) 1998-2021 Paulo Moura.
This HTML page was made with roffit.
logtalk_allure_report [-d tests] [-i results] [-o report] [-t title] [-p] [-- environment pairs]
logtalk_allure_report -h
Description
Options
Exit status
Examples
logtalk_allure_report
allure open
allure open
allure open "$HOME/my_project/allure_report"
Implementation notes
See also
Author
Copyright


{% extends "_base.html" %} {% block titlecontent -%} {%- if grp -%} {{ grp.name }} —. {% endif -%} Packages by Code Stats – {{ super() }} {%- endblock titlecontent %} {% block favicon -%} {%- endblock favicon %} {% block style %} {{ super() }} .loc-th-num { text-align: right } .loc-th-nums { text-align: center } .loc-td-num { text-align: right; } .loc-td-percent { text-align: right; } {% endblock style %} {% macro table_row(pkg) -%}
Here are the unported packages, with the approximate size of their codebases (in lines of code), and the sizes of their Python parts.
The size of the Python parts is split between Python code and C/C++ code using the CPython API.
The information is inaccurate and incomplete, but hopefully useful.
As for the charts: don't compare the “blue areas” between two graphs. The filled-in areas represent the amount of Python/C-API code relative to all code in the particular project. The size of the bar itself uses a logarithmic scale. What the graph can tell you at a glance is: “A {big/medium/small} project, {all/half/hardly} written in Python”.
| {{ sort_link('name', 'Package') }} | {{ sort_link('loc', 'Total') }} | {{ sort_link('python', 'Python') }} | {{ sort_link('capi', 'C API') }} |
|---|


Organize epic beer pong tournament with your friends
Try this new free service to organise epic Beer Pong tournaments
Typical Beer Pong House Rules around the world
A history of two players of Beer Pong


Roxo Hugo is a digital agency Hugo theme for creative agencies, freelancers, graphic designers, photographers. Can be used for any kind of corporate websites who provides digital services with many expertise such as Design, Digital Products, Development, Campaign & Content, Employer Branding, Animation & Motion Graphics ,Packaging & Product Design, Retail & Spacial, Print & Editorial Design, Concept/Text, Information Design etc. Roxo Hugoâ€™s contact form is supported Formspree.
Checkout the live demo here
git submodule add git@github.com:StaticMania/roxo-hugo.git themes/roxo
.Copy the data
, content
, static
, resources
& config.toml
files from the exampleSite
directory and paste it on you Hugo Project repository/directory. From the site home directory:
cp -a themes/roxo/exampleSite/* .
Build your site with hugo serve
and see the result at http://localhost:1313/
.
Have some question or facing any technical trouble feel free to Contact Us
This Repository is licensed under the MIT License
Need help to build HUGO websites with your custom requirements. Feel free to contact with us. We provide custom development service for HUGO.


Hello guys ,welcome to our cookies world . this is our new website project ,and we hope that you will be a member of our family. Mr Pat he is the owner he come up with a business idea... by combining two signature Seattle icons: Pat has developed a recipe for a coffee-time confection called Salmon Cookies.
Why its called salmon cookie ?
because these cookies made into the shape of a salmon that has just a hint of fishy goodness.


Taking Notes
I've carrired a bound, grid paper engineering notebook for quite a few years now. (A habit I picked up in school from the Electrical Engineering side of my formal education.) I've written an entry for (almost) every day working.
There are a few reasons why this has been my preference to text-based note taking, for example: a simple text file, or more complex, a wiki
Low-tech solution for a low-tech problem🔗
If I need to write a to-do list, or a reminder for myself while I'm away from my desk, it is a lot easier to jot something down and be done. I don't require to be near a text-editor. I don't need to create a wiki page, or do any post-markup to what I've written.
Also, I'm free to use a page however I need to in order to get my thought onto the page. I'm not limited to software features for my visual metaphors if I'm sketching out a diagram, or using common symbols
The downsides🔗
This approach has bitten me a few times. When I've started in with a new notebook in the middle of a project (since bound books have limited pages) I'm stuck carrying 2 books for a while.
Text searching features don't exist in my books. At least not without scanning and OCR (Making the assumtion that my handwriting is neat enough to be read into a text form, which is an unrealistic assumption.)
Looking for a balance🔗
It got me thinking about what I needed to do if I wanted to go all digital with my note taking, but keep my workflow intact.
The feeling of using a softer tip on a phone screen requires too much of a change to how I naturally write. I end up concentrating more just to make sure I write legibly enough for myself. Recently, I bought a hard-tipped stylus for my phone, since my experiences with other rubber tipped styli were underwhelming. An argument can be made about the price for a good styli. The good reviews start at about $20-30. A large commitment for an instrument that is only good on touch screens.
I use an Android phone, and, at least at the time of this writing, there aren't a lot of great hand writing tools. Most are marketed towards the digital artist. Lots of authoring features I don't need, and some that are missing for creating journal entries.
Fiinote so far has been my favorite. I get a good mix of hand-written options, hand-drawn, text-based. And I can create hyperlinks, and embed images if I choose to. It has only been a few days, but if the free version is this great, the obvious choice may be to support the developers by upgrading to the full version.
Conclusion🔗
I recommend writing your notes down. There have been many articles and studies about the cognitive advantages of longhand vs keyboarding for note taking.
Hard-tipped styli are pretty great, and cost about as much as nicer ink pens (~$25-100+).
Good, free software for handwriting exists for handwriting. Great, paid software undoubtedly exists.
Here's to a successful experiment!


JSimpleDB is a better persistence layer for Java
Mainstream persistence solutions such as JPA and JDO fail to address many important issues that are inherent to persistence programming. This is because they were not designed to address these issues; they were designed merely to give Java programmers access to existing database functionality.
In particular:
- Configuration complexity Do we have to explicitly configure how data is mapped? Are we forced to (ab)use the programming language to address what are really database configuration issues?
- Query language concordance Does the code that performs queries look like regular Java code, or do we have to learn a new “query language”?
- Query performance transparency Is the performance of a query obvious from looking at the code that performs it?
- Data type congruence Do database types agree with Java types? Are all field values supported? Do we always read back the same values we write?
- First class offline data Can it be precisely defined which data is actually copied out of a transaction? Does offline data have all the rights and privileges of “online” (i.e., transactional) data? Does this include index queries and a framework for handling schema differences?
- Schema verification Is the schema assumed by the code cross-checked against the schema actually present in the database?
- Incremental schema evolution Can multiple schemas exist at the same time in the database, to support rolling upgrades? Can data be migrated incrementally, i.e., without stopping the world? Are any whole database operations ever required?
- Structural schema changes Are structural schema updates performed automatically?
- Semantic schema changes Is there a convenient way to specify semantic schema updates, preferably at the Java level, not the database level? Do semantic updates have access to both the old and the new values?
- Schema evolution type safety Is type safety and data type congruence guaranteed across arbitrary schema migrations?
- Transactional validation Does validation only occur at the end of the transaction, or randomly and inconveniently in the middle?
- Cross-object validation Is it possible to define validation constraints that span multiple objects/records?
- Language-level data maintainability Can data maintenance tasks be performed using the normal Java types and values? Are there convenient tools for manual and scripted use?
JSimpleDB addresses all of these issues, this without sacrificing flexibility or scalability.
JSimpleDB does this by treating the database as just a sorted key/value store, and implementing the following in Java:
- Encoding/decoding of field values
- Enforcing referential integrity; forward/reverse delete cascades
- Field indexes (simple and composite)
- Query views
- Schema management
- Change notification
- Validation queues
- Command line interface
JSimpleDB also adds some new features that traditional databases don't provide.
- Designed from the ground up to be Java-centric; 100% type-safe at all times.
- Works on top of any database that can function as a key/value store (SQL, NoSQL, etc.)
- Scales gracefully to large data sets; no "whole database" operation is ever required
- Configured entirely via Java annotations (only one is required)
- Queries are regular Java code - there is no "query language" needed
- Change notifications from arbitrarily distant objects
- Built-in support for rolling schema changes across multiple nodes with no downtime
- Supports simple and composite indexes, including on user-defined custom types
- Extensible command line interface (CLI) including Java 8 expression parser
- Built-in Java-aware graphical user interface (GUI) based on Vaadin
JSimpleDB Paper
The paper JSimpleDB: Language-Driven Persistence for Java describes the issues that are inherent to persistence programming and how JSimpleDB addresses them.
Abstract:
Most software applications require durable persistence of data. From a programmer’s point of view, persistence has its own set of inherent issues, e.g., how to manage schema changes, yet such issues are rarely addressed in the programming language itself. Instead, how we program for persistence has traditionally been driven by the storage technology side, resulting in incomplete and/or technology-specific support for managing those issues.
In Java, the mainstream solution for basic persistence is the Java Persistence API (JPA). While popular, it also measures poorly on how well it addresses many of these inherent issues. We identify several examples, and generalize them into criteria for evaluating how well any solution serves the programmer’s persistence needs, in any language. We introduce JSimpleDB, a persistence layer for ordered key/value stores that, by integrating the data encoding, query, and indexing functions, provides a more complete, type-safe, and language-driven framework for managing persistence in Java, and addresses all of the issues we identify.
JSimpleDB Maven Artifacts
JSimpleDB is built into several modular components, including:
| jsimpledb-main |
This is the main artifact and the only one that is strictly required (it will pull in some of the others below as dependencies). |
| jsimpledb-coreapi |
Core API classes |
| jsimpledb-kv |
Key/value store API |
| jsimpledb-kv-* |
Various key/value store implementations. You will need one of these if you are doing anything serious. |
| jsimpledb-spring |
Spring-related functionalty, including a JSimpleDB PlatformTransactionManager implementation. |
| jsimpledb-parse |
Java expression parsing code (used by the CLI) |
| jsimpledb-vaadin |
Vaadin-related functionalty |
| jsimpledb-cli |
Command line interface (CLI) classes |
| jsimpledb-cliapp |
Command line interface (CLI) application main class |
| jsimpledb-guiapp |
Graphical user interface (GUI) application main class |
For More Information
More information is available on the Github site.


Teaching Assistant
Adolfo Ibáñez University, ChileCourses: Programming I, Physics I and Business Intelligence
Courses: Programming I, Physics I and Business Intelligence
Social action project created by students of the Adolfo Ibáñez University, whose main mission is to generate awareness about the different social problems that afflict Chile as a country, provide support to communities at risk through community building and development projects and mobilize young people students so that as professionals they dedicate themselves to creating a positive impact from their careers and areas of interest.
Some of the social projects that the organization generates: construction of emergency housing, social headquarters, playgrounds, workshops for children and adults of the community in vulnerable places, sharing with people in street situations, among others.
ADMINISTRATIVE FUNCTIONS IN CASTTVV
August 2016 - December 2016 Member of Project's Department
PARTICIPATION IN VOLUNTEERING
Participation in the volunteering that takes place every winter and summer, where public spaces are recovered through the construction of playgrounds, pergolas or soccer fields while generating ties with the community.
July 2016 - Staff Member in charge of construction area in Winter Volunteering in Alhué
January 2016 - Team leader in Summer Volunteering in Marchigüe
July 2015 - Team Leader in Winter Volunteering in Peumo
May 2015 - Team Leader in Emergency Housing constructions in Diego de Almagro
January 2015 - Volunteer in Summer Volunteering in Quillota
June 2014 - Volunteer in Emergency Housing construction in Valparaíso
Worldpackers is a community that connects travelers looking to exchange their skills for accommodation with hosts from all around the world.
I was a volunteer in the following hostels:
February 2018 - March 2018 Cleaning assistant in Maraca Hostel. Rio do Janeiro, Brasil.
January 2019 - end February 2019 Receptionist at ITH Zoo hostel. San Diego, CA.
March 2019 Receptionist at Hostel Kabeh. Cancún, México.


Parkhurst Has The Ball
What should Michael Parkhurst do?
Pass the ball back to Guzan
Pass it to Almiron


Issue a new ID to a participant in your business network
Issuing an identity generates a one-time secret. You can choose to send this to somebody or use it yourself when it has been issued.


Jose Mourinho showers encomium on Dele Alli after Tottenham's win over West Ham
- Jose Mourinho has commended Dele Alli after his side's win over West Ham United
- The Portuguese born tactician guided Tottenham to their first away win in months at the London Stadium
Mourinho said Alli is too good not to be a starter for the north London side and the national team
Tottenham Hotspur new manager Jose Mourinho has showered praises on England midfielder Dele Alli after their 3-2 win over West Ham United at the London Stadium.
The 23-year-old provided a superb assist which led to Spurs opening goal through Son Heung-min in the 36th minute of the encounter.
Lucas Moura and Harry Kane also scored a goal each to ensure the Portuguese tactician claim his first win in charge of the team.
Mourinho said: "I’m happy with him, I spent a few minutes with him in training and outside the pitch. And we were saying that the best Dele Alli has to be back.
Fans rate Ndidi better than £70m England midfielder after impressive performance against Arsenal
PAY ATTENTION: Install Pitch Football App for FREE to get the latest football News & Scores
"He’s too good to not be one of the best players in the world and not playing with the national team.
The Portuguese born tactician said on Alii that: "He has to start from now, has to start now without injuries and start with some stability from tactical point of view.’
Mourinho also revealed his conversation with the said player before this evening's fixture, insisting Alli is too good not to be an important player for Spurs.
"I spent a few minutes with him in training, and also in our social life outside the pitch, and we were speaking that the best Dele has to be back," he told BT Sport.
READ ALSO: NAIJ.com upgrades to Legit.ng
"He is too good to not be in the national team. He is too good to not be a fundamental player for Spurs.
Nigeria crash out of U17 World Cup following disappointing loss to Netherlands
"He is too good to not be one of the best players in the world, so he has to start now to, without injuries, hopefully, have some stability, and he is a fantastic player."
Legit.ng earlier reported that Jose Mourinho got his reign as Tottenham Hotspur head coach on a winning note following his side's 3-2 win over West Ham United at the London Stadium.
The Portuguese took over at the club earlier this week after the sack of former handler Mauricio Pochettino.
A goal each from Lucas Moura, Harry Kane, and Son Heung-min sealed the new manager's first win on the road.
Meanwhile, two second-half goals from Michail Antonio and Angelo Ogbonna failed to save them from losing the game.
Can Nigeria U17 team win a 6th World Cup title in Brazil? | Legit TV


You might have to go to court or pay a penalty if you deliberately give false or incomplete information, or don't report changes. Your ESA may also be stopped or reduced.
You might have to go to court or pay a penalty if you deliberately give false or incomplete information, or don't report changes. Your ESA may also be stopped or reduced.


Organizing Infrastructure Config and Workflow
@ HashiConf 2016 September 6-8, 2016
Video
Slides
Abstract
When starting with the various products Terraform, Packer, Vagrant, and others, it isn’t always apparent where and in what way one should organize the actual project. In this talk I’d like to delve into what I’ve done to organize solutions for development, production, and related pipelines. I’ll talk from my point of view and what I’ve seen others do to keep their workloads organized and their infrastructure and application pipelines clean and well organized.
The Presentation
The Why
- To show, mentor, or teach all of you what we’ve been working on that you would hopefully find useful.
- To speak with you - after or before this talk about what you’re working on that can benefit us, and how we can benefit you. We want this to be win - win.
Home Depot & Google Cloud Platform
Currently I work with Home Depot building out infrastructure and services for what is called Quote Center. This is the custom order application that allows Home Depot to put together custom orders for builders and others that have specific and unique requirements for their orders.
My co-presenter Evan works with the Google Cloud Platform team bringing together key elements around Bosh, Pivotal Cloud Foundry, Terraform, and other pieces for various customers. I’ve had the fortune of working with Evan as a customer of AWS & GCP since he was a chef of awesomesauce at AWS.
The Work Described in Presentation
What & Where
We’re a team that is autonomous from the Home Depot IT Organization, albeit still fully connected, which also gives us unique autonomy to build, scale, and implement unhindered. This allows us to really focus on our specific business units’ needs. This provides us a kind of “laboratory” status here in Cascadia).
What We Have
Our environment is a language and operating system polyglot dream (or mess depending on one’s POV). The mix of operating systems, system workflows, and languages with their respective stacks all around make for a challenge that either defeats the soul or stirs the passion. For me, I get lit up with excitement over getting it to interoperate and smoothly work.
Our core applications do fall along using the Node.js, JavaScript, and .NET Stacks. But beyond that core functionality we have data anlytics projects getting built on Kafka, Zookeeper, and a host of microservice style stacks on various Linux distributions and Windows Servers.
Where We’re Going
Another important situational report I need to provide everyone is the fact most of this infrastructure exists where we don’t want it and we’re in the process of migrating entirely to Google Cloud. This is what the ideas here will involve, is Google Cloud and the work I’m doing to get us migrated to and deploying to Google Cloud.
Our 3 Eras
Legacy “Iron Throne Era”
Some of the key identifying characteristics of systems in this era include:
- System won’t work
- System sort of works
- System can’t move
- System is barely (or not) Internet ready
This is the term we’ve started to use, currently, for the definition of legacy in relation to migration/operation in Google Cloud. We’re fortunate at Home Depot in the Quote Center, in that we generally we don’t have applications in these realms. But, there are a few.
Infrastructure “Renaissance Era”
- Google Compute Engine
- System Applications
This era and system type includes things that are solely focused around the traditional systems administration and deployment style of “a server” with everything built on top. More simply, think of this as systems that run on a singular, or maybe a manually created cluster within Google Compute Engine.
Another aspect of this era, which is easier to define, is an application that is traditionally hosted, built, or maintained. Think of applications like JIRA, Enterprise Github, SQL Server Reporting Services, or even SQL Server itself. These applications or servers on require, and can really only utilize a single instance in most use cases.
These are the systems and applications of the “Renaissance Era”.
 This is where things get fast, exciting, and quality starts to dramatically improve.


Privacy Statement
All your data are belong to us. Use jefago bot at your own risk - it is provided "as is".


Mandatory Sessions
There are some mandatory events for all Freshmores to attend.
These events are also shaded in pink on the calendar.
| Session Name | Location | Date and Time |
|---|---|---|
| Academic Matters Briefing | To Be Determined | Monday, 14 September 2020, 1400-1800hrs |
| Learning Journey | To Be Determined | Tuesday, 15 September 2020, 1400-1600hrs |
| Hostel Briefing | To Be Determined | Tuesday, 15 September 2020, 2000-2100hrs |
| Hostel Floor Supper | To Be Determined | Tuesday, 15 September 2020, 2100-2200hrs |
| English Writing Entrance Test (EWET) - Selected Students Only | To Be Determined | Friday, 18 September 2020, 1500-1630hrs |
| PPP Briefing | To Be Determined | Wednesday, 23 September 2020 |
| PPP Briefing | To Be Determined | Friday, 25 September 2020 |
DiscoverPillars
ASD Pillar Introduction
Organised by ASD
Important Information
- Date: Thursday, 17 September 2020
- Time: 2:00pm to 4:00pm
- Venue: To Be Determined
ESD Pillar Introduction
Organised by ESD
Important Information
- Date: Thursday, 17 September 2020
- Time: 2:00pm to 4:00pm
- Venue: To Be Determined
EPD Pillar Introduction
Organised by EPD
Important Information
- Date: Thursday, 17 September 2020
- Time: 2:00pm to 4:00pm
- Venue: To Be Determined
ISTD Pillar Introduction
Organised by ISTD
Important Information
- Date: Thursday, 17 September 2020
- Time: 2:00pm to 4:00pm
- Venue: To Be Determined
Specialist/Maker Cluster
Coming Soon!
Sports Cluster
Coming Soon!
Arts/Music Cluster
Coming Soon!
Others
Supper
Organised by DiscoverSUTD
Food is a good icebreaker.
At SUTD, we love free food, so that's what we want to share with our juniors!
We'll be arranging for a variety of snacks and finger food to be available twice a week during
the DiscoverSUTD period. We hope that you'll drop by for a quick bite and engage in
conversations with your fellow peers.
On each day, there will be one vegetarian and one non-vegetarian food item.
- Time: 8:00pm to 11:00pm
- Date: 16, 17, 21, 22, 24, 28, 29 September 2020 and 1 October 2020
- Venue: To Be Determined
Human Library
Organised by DiscoverSUTD
Come Forth and DiscoverPeople
Human Library is a place where real people ("books") are on loan to readers (you!).
Here, you can meet your seniors and faculty, and ask them about their experiences in and
outside of SUTD. Take a look at who will be there!
- Date: Wednesday, 16 September 2020
- Time: 2:00pm to 4:00pm
- Venue: To Be Determined
- Booklist: Book Bios
DiscoverPlaces
Organised by DiscoverSUTD
Come Forth and Discover Our Labs and Facilities
SUTD has many research labs located all around the campus. These labs are usually not accessible
due to research work but for one day only, we will be conducting lab tours!
| Lab Visit Summary Table | |||
| Lab Name | Meeting Point/Time | Contact Person | Sign-Up Link |
|---|---|---|---|
| To Be Determined | To Be Determined | To Be Determined | To Be Determined |
- Date: Thursday, 17 September 2020
- Time: 4:00pm to 5:30pm
- Venue: Various


Drag & Drop functionality in AngularJS, no jQuery required
Simple and easy to use tools for building drag & drop directives, or integrating drag & drop functionality in your own directives, with a very simple and easy to use API.
Code on Github Download (<%= pkg.version%>) API Documentation
If dropped into one of these three areas, I will snap into place. Otherwise, I'll just hang around.


Intro
I was walking home the other day this week, looking at the various cars on the street German, Japanese, American and so on and it got me thinking. What is it that I find more visually appealing about German and Japanese cars? Why do I react differently to cars from certain countries? Is there something fundamental about British styling, Italian styling, and so on? Google not too long ago open-sourced a pre-trained, 40+ layer deep neural network called Inception. Alongside, there is a tutorial, Tensorflow for Poets, which shows how to use this network and train the last layer so you can have it classify between your very own image datasets!
Methods
I followed the tutorial and then tried it again with a data set of about 500 images, about a fifth from each of five car origins: Germany, Japan, United States, Italy and Great Britain. There are some Google Chrome extensions you can use to bulk download Google Images results. I had to clean up what I downloaded and delete some photos that were out of place. The photos are about 10-20 kb in size, which makes them faster to train with.
Results
While training with flower data set given in the tutorial yieled around 90% test accuracy, the network achieved just 73.1% on my much smaller dataset of car photos. Keep in mind, if one were to make random guesses, they would most likely average 20% accuracy. Below are some additional test results for some novel downloaded images, 2 from each car origin. The network does make some funny predictions, but other predictions are very good:
2017-07-22 04:57:55.607929: Step 490: Train accuracy = 76.0%
2017-07-22 04:57:55.608473: Step 490: Cross entropy = 0.764508
2017-07-22 04:57:55.916217: Step 490: Validation accuracy = 58.0% (N=100)
2017-07-22 04:57:58.682838: Step 499: Train accuracy = 74.0%
2017-07-22 04:57:58.683158: Step 499: Cross entropy = 0.778337
2017-07-22 04:57:58.989198: Step 499: Validation accuracy = 60.0% (N=100)
Final test accuracy = 73.1% (N=52)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
Discussion
This neural network is not going to outsmart a car fanatic quite yet. However, for a network trained on around 500 images, and all hyperparameters left untouched, it is quite impressive that it can guess better than 3x what random guessing would result in. Having cars from a very large time span added to the network's difficulty in making predictions. Design languages are not mutually exclusive from origin to origin so it can be difficult even for a car fanatic to make some distinctions. For example, the Kia Motors aquired a German designer to design the Kia Optima a few years back and it naturally had some German design cues.
Conclusion
There is certainly a correlation between the country of origin of various car and their respective design cues. The network could not simply read the badges for the various cars, so it had to recognize actual design features. The network did not do a very good job, but there is plenty of room for improvement.
Future work
Since neural networks always work better with more data, adding more training examples would certainly yield better results. The hyperparameters could be tweaked. We could see how a simpler convolutional neural network with fewer layers would fare (this would require going beyond simply retraining Inception and creating our own network). The dataset could also be refined and cleaned to include only cars from a specified era to see if this would give better results.


It is important for the United States to participate in international organizations that support human rights and that hold individuals accountable for mass atrocities


We quote fresh from wikipedia:
Internet security is a branch of computer security specifically related to the Internet, often involving browser security but also network security on a more general level as it applies to other applications or operating systems on a whole. Its objective is to establish rules and measures to use against attacks over the Internet.The Internet represents an insecure channel for exchanging information leading to a high risk of intrusion or fraud, such as phishing.Different methods have been used to protect the transfer of data, including encryption and from-the-ground-up engineering.
Phishing is the attempt to obtain sensitive information such as usernames, passwords, and credit card details (and, indirectly, money), often for malicious reasons, by disguising as a trustworthy entity in an electronic communication.
| A legitimate example of phising.(source) |
Phishing is a major problem because there really is no patch for human stupidity


{% extends 'all-images/base.html' %} {% block content %} {{block.super}} Personal Photoset IT SEASONS. BEARING IN ONE YEARS FORTH SAYING View Gallery Now Location Filter All {% for location in locations %} {{location.location}} {% endfor %} {% for photo in photos %} {{photo.name}} View Photo {% empty %} NO Image Available {% endfor %} {% endblock %}
IT SEASONS. BEARING IN ONE YEARS FORTH SAYING


Join our TeaCup community and learn new technologies,
share your knowledge with the colleagues and meet the best specialists.
Join our TeaCup community and learn new technologies,
share your knowledge with the colleagues and meet the best specialists.


Pactia+
Indicadores
Sign in
You may also need to provision your
BIM 360 Docs account for this app.
Learn more
.


UKCloud Limited (“UKC”) and Virtual Infrastructure Group Limited (“VIG”) (together “the Companies”) – in Compulsory Liquidation
On 25 October 2022, the Companies were placed into Liquidation with the Official Receiver appointed as Liquidator and J Robinson and A M Hudson simultaneously appointed as Special Managers to manage the liquidation process on behalf of the Official Receiver.
Further information regarding the Liquidations can be found here: https://www.gov.uk/government/news/virtual-infrastructure-group-limited-and-ukcloud-limited-information-for-creditors-and-interested-parties
Contact details:
For any general queries relating to the Liquidations please email<EMAIL_ADDRESS>For customer related queries please email<EMAIL_ADDRESS>For supplier related queries please email<EMAIL_ADDRESS>UKCloud for Red Hat OpenShift
Efficiently refactor, rearchitect or build new portable cloud native applications
As a developer-focused Platform-as-a-Service (PaaS) offering, UKCloud for Red Hat OpenShift helps to increase the agility and efficiency of your software development lifecycle – giving you the tools you need to adopt DevOps best practice. By divorcing development and application delivery from infrastructure, you can start to move your IT teams up the technology stack and start to focus on longer-term business value and strategic projects.
Combining our secure and sovereign cloud with Red Hat's award-winning open source container platform, you can efficiently progress your modernisation journey from software to services, through to delivering innovative digital services and cloud-native applications.
Let's get started!
For a general product overview, including benefits, product options and packaging information, see the Service Definition. If you need more in-depth product details, see the Service Scope.
The Getting Started Guide provides an overview of the functionality available with UKCloud for Red Hat OpenShift and introduces some of the key concepts and tasks, including the different ways to interact with the service.
Key topics
Once you understand the basics of UKCloud for Red Hat OpenShift, take a look at the articles listed below to get an idea of what else you can do.
Managing your environment
Working with networks
Backup and recovery
Build an OpenShift application from a private GitHub repo
Expand OpenShift persistent volumes
Lifecycle and patch management
Making the most of your UKCloud for Red Hat OpenShift trial
Use NetworkPolicy objects in OpenShift v4
Deploying an application into OpenShift Service Mesh (v4.x)
DNS zone forwarding in OpenShift clusters (v4.x)
Community network proxy configuration (v3.x)
Customise the router error page (v3.x)
Expose non-http services externally (v3.x)
Permit outbound internet access in deployments with government community network connectivity (v3.x)
Publish application routes to multiple external networks (v3.x)
Monitoring and logging
Security
Want to know more?
Find answers to common questions in our UKCloud for Red Hat OpenShift FAQ. Use the table of contents on the left to browse all UKCloud for Red Hat OpenShift articles or use the search box at the top to search the whole Knowledge Centre.
For more information about UKCloud and how we work, browse our website.
How to buy
The UKCloud Pricing Guide lists all our product pricing. For examples of how pricing works for UKCloud for Red Hat OpenShift, take a look at our Pricing information article.
If you're interested in buying UKCloud for Red Hat OpenShift, or would like a quote:
If you're already a UKCloud customer
Raise a Service Request, contact your Service Delivery Manager or email<EMAIL_ADDRESS>If you're new to UKCloud
Contact us via our website or call us on 01252 303 300
Additional options
To get the most out of our platform, see below for some additional options and products you might want to consider using with UKCloud for Red Hat OpenShift (additional charges apply).
Product options
Complementary products
Additional Control Plane Nodes. An operational cluster requires a minimum of three Control Plane Nodes. Depending on your workloads or solution, you may need more.
Additional Infrastructure Nodes. Infrastructure Nodes are not mandatory, but we recommend a minimum of two to provide integrated cluster services. You may want more if you require additional services.
Additional Application Nodes. You need at least one Application Node to run your application, however we recommend a minimum of two nodes for resilience. You may need more for duplicated or new applications.
Registry storage. The OpenShift image registry persistent storage is hosted on UKCloud Cloud Storage.
Tier 1 storage. Improved performance for workloads requiring higher disk throughput and IOPS.
Managed IT Operations. Reduce the amount of time spent on managing your IT infrastructure.
Security Operations Service. People, processes and tooling to assist with cyber monitoring capabilities.
Professional Services. Help with solution design and digitisation projects.
Private Cloud. For workloads that require custom configurations, such as large instances, or complex licensing requirements.
Cloud Storage. Cost effective unstructured object storage for data archive.
If you want to find out more about any of these options or products, contact your Service Delivery Manager or email<EMAIL_ADDRESS>Why UKCloud?
The right cloud for every platform. Modernise legacy workloads across the board by rehosting traditional systems with UKCloud for VMware or exploring new and emerging technologies with OpenShift. We can help you build a cloud specific to your needs.
An award-winning customer experience. The customer sits at the heart of everything we do. Whether that's through our 24/7 support function – a service we provide at no extra charge – or by giving every single one of our customers a dedicated Service Delivery Manager.
Dedicated to the public sector. From our unique assurance and connectivity options to the people we employ – everything we do comes back to our mission to help public sector organisations deliver better digital services at a lower cost.
Uncompromising security and integrity. Our UK-based data centres help protect your organisation from regulations such as GDPR, whilst our unique assurance and connectivity options enable you to move your most secure and sensitive systems to the cloud.
Champions of innovation. We've partnered with over 300 specialist organisations who have built their pioneering digital services on top of our government-grade infrastructure. From AI to RPA, our partner ecosystem has you covered.
Committed to doing the right thing. From paying our taxes to undergraduate and apprentice programmes. From Carbon Neutral solutions to charity work. Everything we do can be traced back to our commitment to doing the right thing.


Sometimes file uploaders rely on Hidden Segments of IIS Request Filtering such as APP_Data or App_GlobalResources directories to make the uploaded files inaccessible directly.
However, this method can be bypassed by removing the hidden segments by using the following web.config file:
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<system.webServer>
<security>
<requestFiltering>
<hiddenSegments>
<remove segment="bin" />
<remove segment="App_code" />
<remove segment="App_GlobalResources" />
<remove segment="App_LocalResources" />
<remove segment="App_Browsers" />
<remove segment="App_WebReferences" />
<remove segment="App_Data" />
<!--Other IIS hidden segments can be listed here -->
</hiddenSegments>
</requestFiltering>
</security>
</system.webServer>
</configuration>
Now, an uploaded web shell file can be directly accessible.