source file: /Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/s3transfer/utils.py
file stats: 348 lines, 97 executed: 27.9% covered
1. # Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2. # 3. # Licensed under the Apache License, Version 2.0 (the "License"). You 4. # may not use this file except in compliance with the License. A copy of 5. # the License is located at 6. # 7. # http://aws.amazon.com/apache2.0/ 8. # 9. # or in the "license" file accompanying this file. This file is 10. # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11. # ANY KIND, either express or implied. See the License for the specific 12. # language governing permissions and limitations under the License. 13. import random 14. import time 15. import functools 16. import math 17. import os 18. import stat 19. import string 20. import logging 21. import threading 22. import io 23. from collections import defaultdict 24. 25. from s3transfer.compat import rename_file 26. from s3transfer.compat import seekable 27. 28. 29. MAX_PARTS = 10000 30. # The maximum file size you can upload via S3 per request. 31. # See: http://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html 32. # and: http://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html 33. MAX_SINGLE_UPLOAD_SIZE = 5 * (1024 ** 3) 34. MIN_UPLOAD_CHUNKSIZE = 5 * (1024 ** 2) 35. logger = logging.getLogger(__name__) 36. 37. 38. def random_file_extension(num_digits=8): 39. return ''.join(random.choice(string.hexdigits) for _ in range(num_digits)) 40. 41. 42. def signal_not_transferring(request, operation_name, **kwargs): 43. if operation_name in ['PutObject', 'UploadPart'] and \ 44. hasattr(request.body, 'signal_not_transferring'): 45. request.body.signal_not_transferring() 46. 47. 48. def signal_transferring(request, operation_name, **kwargs): 49. if operation_name in ['PutObject', 'UploadPart'] and \ 50. hasattr(request.body, 'signal_transferring'): 51. request.body.signal_transferring() 52. 53. 54. def calculate_range_parameter(part_size, part_index, num_parts, 55. total_size=None): 56. """Calculate the range parameter for multipart downloads/copies 57. 58. :type part_size: int 59. :param part_size: The size of the part 60. 61. :type part_index: int 62. :param part_index: The index for which this parts starts. This index starts 63. at zero 64. 65. :type num_parts: int 66. :param num_parts: The total number of parts in the transfer 67. 68. :returns: The value to use for Range parameter on downloads or 69. the CopySourceRange parameter for copies 70. """ 71. # Used to calculate the Range parameter 72. start_range = part_index * part_size 73. if part_index == num_parts - 1: 74. end_range = '' 75. if total_size is not None: 76. end_range = str(total_size - 1) 77. else: 78. end_range = start_range + part_size - 1 79. range_param = 'bytes=%s-%s' % (start_range, end_range) 80. return range_param 81. 82. 83. def get_callbacks(transfer_future, callback_type): 84. """Retrieves callbacks from a subscriber 85. 86. :type transfer_future: s3transfer.futures.TransferFuture 87. :param transfer_future: The transfer future the subscriber is associated 88. to. 89. 90. :type callback_type: str 91. :param callback_type: The type of callback to retrieve from the subscriber. 92. Valid types include: 93. * 'queued' 94. * 'progress' 95. * 'done' 96. 97. :returns: A list of callbacks for the type specified. All callbacks are 98. preinjected with the transfer future. 99. """ 100. callbacks = [] 101. for subscriber in transfer_future.meta.call_args.subscribers: 102. callback_name = 'on_' + callback_type 103. if hasattr(subscriber, callback_name): 104. callbacks.append( 105. functools.partial( 106. getattr(subscriber, callback_name), 107. future=transfer_future 108. ) 109. ) 110. return callbacks 111. 112. 113. def invoke_progress_callbacks(callbacks, bytes_transferred): 114. """Calls all progress callbacks 115. 116. :param callbacks: A list of progress callbacks to invoke 117. :param bytes_transferred: The number of bytes transferred. This is passed 118. to the callbacks. If no bytes were transferred the callbacks will not 119. be invoked because no progress was achieved. It is also possible 120. to receive a negative amount which comes from retrying a transfer 121. request. 122. """ 123. # Only invoke the callbacks if bytes were actually transferred. 124. if bytes_transferred: 125. for callback in callbacks: 126. callback(bytes_transferred=bytes_transferred) 127. 128. 129. def get_filtered_dict(original_dict, whitelisted_keys): 130. """Gets a dictionary filtered by whitelisted keys 131. 132. :param original_dict: The original dictionary of arguments to source keys 133. and values. 134. :param whitelisted_key: A list of keys to include in the filtered 135. dictionary. 136. 137. :returns: A dictionary containing key/values from the original dictionary 138. whose key was included in the whitelist 139. """ 140. filtered_dict = {} 141. for key, value in original_dict.items(): 142. if key in whitelisted_keys: 143. filtered_dict[key] = value 144. return filtered_dict 145. 146. 147. class CallArgs(object): 148. def __init__(self, **kwargs): 149. """A class that records call arguments 150. 151. The call arguments must be passed as keyword arguments. It will set 152. each keyword argument as an attribute of the object along with its 153. associated value. 154. """ 155. for arg, value in kwargs.items(): 156. setattr(self, arg, value) 157. 158. 159. class FunctionContainer(object): 160. """An object that contains a function and any args or kwargs to call it 161. 162. When called the provided function will be called with provided args 163. and kwargs. 164. """ 165. def __init__(self, func, *args, **kwargs): 166. self._func = func 167. self._args = args 168. self._kwargs = kwargs 169. 170. def __repr__(self): 171. return 'Function: %s with args %s and kwargs %s' % ( 172. self._func, self._args, self._kwargs) 173. 174. def __call__(self): 175. return self._func(*self._args, **self._kwargs) 176. 177. 178. class CountCallbackInvoker(object): 179. """An abstraction to invoke a callback when a shared count reaches zero 180. 181. :param callback: Callback invoke when finalized count reaches zero 182. """ 183. def __init__(self, callback): 184. self._lock = threading.Lock() 185. self._callback = callback 186. self._count = 0 187. self._is_finalized = False 188. 189. @property 190. def current_count(self): 191. with self._lock: 192. return self._count 193. 194. def increment(self): 195. """Increment the count by one""" 196. with self._lock: 197. if self._is_finalized: 198. raise RuntimeError( 199. 'Counter has been finalized it can no longer be ' 200. 'incremented.' 201. ) 202. self._count += 1 203. 204. def decrement(self): 205. """Decrement the count by one""" 206. with self._lock: 207. if self._count == 0: 208. raise RuntimeError( 209. 'Counter is at zero. It cannot dip below zero') 210. self._count -= 1 211. if self._is_finalized and self._count == 0: 212. self._callback() 213. 214. def finalize(self): 215. """Finalize the counter 216. 217. Once finalized, the counter never be incremented and the callback 218. can be invoked once the count reaches zero 219. """ 220. with self._lock: 221. self._is_finalized = True 222. if self._count == 0: 223. self._callback() 224. 225. 226. class OSUtils(object): 227. def get_file_size(self, filename): 228. return os.path.getsize(filename) 229. 230. def open_file_chunk_reader(self, filename, start_byte, size, callbacks): 231. return ReadFileChunk.from_filename(filename, start_byte, 232. size, callbacks, 233. enable_callbacks=False) 234. 235. def open_file_chunk_reader_from_fileobj(self, fileobj, chunk_size, 236. full_file_size, callbacks, 237. close_callbacks=None): 238. return ReadFileChunk( 239. fileobj, chunk_size, full_file_size, 240. callbacks=callbacks, enable_callbacks=False, 241. close_callbacks=close_callbacks) 242. 243. def open(self, filename, mode): 244. return open(filename, mode) 245. 246. def remove_file(self, filename): 247. """Remove a file, noop if file does not exist.""" 248. # Unlike os.remove, if the file does not exist, 249. # then this method does nothing. 250. try: 251. os.remove(filename) 252. except OSError: 253. pass 254. 255. def rename_file(self, current_filename, new_filename): 256. rename_file(current_filename, new_filename) 257. 258. def is_special_file(cls, filename): 259. """Checks to see if a file is a special UNIX file. 260. 261. It checks if the file is a character special device, block special 262. device, FIFO, or socket. 263. 264. :param filename: Name of the file 265. 266. :returns: True if the file is a special file. False, if is not. 267. """ 268. # If it does not exist, it must be a new file so it cannot be 269. # a special file. 270. if not os.path.exists(filename): 271. return False 272. mode = os.stat(filename).st_mode 273. # Character special device. 274. if stat.S_ISCHR(mode): 275. return True 276. # Block special device 277. if stat.S_ISBLK(mode): 278. return True 279. # Named pipe / FIFO 280. if stat.S_ISFIFO(mode): 281. return True 282. # Socket. 283. if stat.S_ISSOCK(mode): 284. return True 285. return False 286. 287. 288. class DeferredOpenFile(object): 289. def __init__(self, filename, start_byte=0, mode='rb', open_function=open): 290. """A class that defers the opening of a file till needed 291. 292. This is useful for deferring opening of a file till it is needed 293. in a separate thread, as there is a limit of how many open files 294. there can be in a single thread for most operating systems. The 295. file gets opened in the following methods: ``read()``, ``seek()``, 296. and ``__enter__()`` 297. 298. :type filename: str 299. :param filename: The name of the file to open 300. 301. :type start_byte: int 302. :param start_byte: The byte to seek to when the file is opened. 303. 304. :type mode: str 305. :param mode: The mode to use to open the file 306. 307. :type open_function: function 308. :param open_function: The function to use to open the file 309. """ 310. self._filename = filename 311. self._fileobj = None 312. self._start_byte = start_byte 313. self._mode = mode 314. self._open_function = open_function 315. 316. def _open_if_needed(self): 317. if self._fileobj is None: 318. self._fileobj = self._open_function(self._filename, self._mode) 319. if self._start_byte != 0: 320. self._fileobj.seek(self._start_byte) 321. 322. @property 323. def name(self): 324. return self._filename 325. 326. def read(self, amount=None): 327. self._open_if_needed() 328. return self._fileobj.read(amount) 329. 330. def write(self, data): 331. self._open_if_needed() 332. self._fileobj.write(data) 333. 334. def seek(self, where): 335. self._open_if_needed() 336. self._fileobj.seek(where) 337. 338. def tell(self): 339. if self._fileobj is None: 340. return self._start_byte 341. return self._fileobj.tell() 342. 343. def close(self): 344. if self._fileobj: 345. self._fileobj.close() 346. 347. def __enter__(self): 348. self._open_if_needed() 349. return self 350. 351. def __exit__(self, *args, **kwargs): 352. self.close() 353. 354. 355. class ReadFileChunk(object): 356. def __init__(self, fileobj, chunk_size, full_file_size, 357. callbacks=None, enable_callbacks=True, close_callbacks=None): 358. """ 359. 360. Given a file object shown below:: 361. 362. |___________________________________________________| 363. 0 | | full_file_size 364. |----chunk_size---| 365. f.tell() 366. 367. :type fileobj: file 368. :param fileobj: File like object 369. 370. :type chunk_size: int 371. :param chunk_size: The max chunk size to read. Trying to read 372. pass the end of the chunk size will behave like you've 373. reached the end of the file. 374. 375. :type full_file_size: int 376. :param full_file_size: The entire content length associated 377. with ``fileobj``. 378. 379. :type callbacks: A list of function(amount_read) 380. :param callbacks: Called whenever data is read from this object in the 381. order provided. 382. 383. :type enable_callbacks: boolean 384. :param enable_callbacks: True if to run callbacks. Otherwise, do not 385. run callbacks 386. 387. :type close_callbacks: A list of function() 388. :param close_callbacks: Called when close is called. The function 389. should take no arguments. 390. """ 391. self._fileobj = fileobj 392. self._start_byte = self._fileobj.tell() 393. self._size = self._calculate_file_size( 394. self._fileobj, requested_size=chunk_size, 395. start_byte=self._start_byte, actual_file_size=full_file_size) 396. self._amount_read = 0 397. self._callbacks = callbacks 398. if callbacks is None: 399. self._callbacks = [] 400. self._callbacks_enabled = enable_callbacks 401. self._close_callbacks = close_callbacks 402. if close_callbacks is None: 403. self._close_callbacks = close_callbacks 404. 405. @classmethod 406. def from_filename(cls, filename, start_byte, chunk_size, callbacks=None, 407. enable_callbacks=True): 408. """Convenience factory function to create from a filename. 409. 410. :type start_byte: int 411. :param start_byte: The first byte from which to start reading. 412. 413. :type chunk_size: int 414. :param chunk_size: The max chunk size to read. Trying to read 415. pass the end of the chunk size will behave like you've 416. reached the end of the file. 417. 418. :type full_file_size: int 419. :param full_file_size: The entire content length associated 420. with ``fileobj``. 421. 422. :type callbacks: function(amount_read) 423. :param callbacks: Called whenever data is read from this object. 424. 425. :type enable_callbacks: bool 426. :param enable_callbacks: Indicate whether to invoke callback 427. during read() calls. 428. 429. :rtype: ``ReadFileChunk`` 430. :return: A new instance of ``ReadFileChunk`` 431. 432. """ 433. f = open(filename, 'rb') 434. f.seek(start_byte) 435. file_size = os.fstat(f.fileno()).st_size 436. return cls(f, chunk_size, file_size, callbacks, enable_callbacks) 437. 438. def _calculate_file_size(self, fileobj, requested_size, start_byte, 439. actual_file_size): 440. max_chunk_size = actual_file_size - start_byte 441. return min(max_chunk_size, requested_size) 442. 443. def read(self, amount=None): 444. if amount is None: 445. amount_to_read = self._size - self._amount_read 446. else: 447. amount_to_read = min(self._size - self._amount_read, amount) 448. data = self._fileobj.read(amount_to_read) 449. self._amount_read += len(data) 450. if self._callbacks is not None and self._callbacks_enabled: 451. invoke_progress_callbacks(self._callbacks, len(data)) 452. return data 453. 454. def signal_transferring(self): 455. self.enable_callback() 456. if hasattr(self._fileobj, 'signal_transferring'): 457. self._fileobj.signal_transferring() 458. 459. def signal_not_transferring(self): 460. self.disable_callback() 461. if hasattr(self._fileobj, 'signal_not_transferring'): 462. self._fileobj.signal_not_transferring() 463. 464. def enable_callback(self): 465. self._callbacks_enabled = True 466. 467. def disable_callback(self): 468. self._callbacks_enabled = False 469. 470. def seek(self, where): 471. self._fileobj.seek(self._start_byte + where) 472. if self._callbacks is not None and self._callbacks_enabled: 473. # To also rewind the callback() for an accurate progress report 474. invoke_progress_callbacks( 475. self._callbacks, bytes_transferred=where - self._amount_read) 476. self._amount_read = where 477. 478. def close(self): 479. if self._close_callbacks is not None and self._callbacks_enabled: 480. for callback in self._close_callbacks: 481. callback() 482. self._fileobj.close() 483. 484. def tell(self): 485. return self._amount_read 486. 487. def __len__(self): 488. # __len__ is defined because requests will try to determine the length 489. # of the stream to set a content length. In the normal case 490. # of the file it will just stat the file, but we need to change that 491. # behavior. By providing a __len__, requests will use that instead 492. # of stat'ing the file. 493. return self._size 494. 495. def __enter__(self): 496. return self 497. 498. def __exit__(self, *args, **kwargs): 499. self.close() 500. 501. def __iter__(self): 502. # This is a workaround for http://bugs.python.org/issue17575 503. # Basically httplib will try to iterate over the contents, even 504. # if its a file like object. This wasn't noticed because we've 505. # already exhausted the stream so iterating over the file immediately 506. # stops, which is what we're simulating here. 507. return iter([]) 508. 509. 510. class StreamReaderProgress(object): 511. """Wrapper for a read only stream that adds progress callbacks.""" 512. def __init__(self, stream, callbacks=None): 513. self._stream = stream 514. self._callbacks = callbacks 515. if callbacks is None: 516. self._callbacks = [] 517. 518. def read(self, *args, **kwargs): 519. value = self._stream.read(*args, **kwargs) 520. invoke_progress_callbacks(self._callbacks, len(value)) 521. return value 522. 523. 524. class NoResourcesAvailable(Exception): 525. pass 526. 527. 528. class TaskSemaphore(object): 529. def __init__(self, count): 530. """A semaphore for the purpose of limiting the number of tasks 531. 532. :param count: The size of semaphore 533. """ 534. self._semaphore = threading.Semaphore(count) 535. 536. def acquire(self, tag, blocking=True): 537. """Acquire the semaphore 538. 539. :param tag: A tag identifying what is acquiring the semaphore. Note 540. that this is not really needed to directly use this class but is 541. needed for API compatibility with the SlidingWindowSemaphore 542. implementation. 543. :param block: If True, block until it can be acquired. If False, 544. do not block and raise an exception if cannot be aquired. 545. 546. :returns: A token (can be None) to use when releasing the semaphore 547. """ 548. logger.debug("Acquiring %s", tag) 549. if not self._semaphore.acquire(blocking): 550. raise NoResourcesAvailable("Cannot acquire tag '%s'" % tag) 551. 552. def release(self, tag, acquire_token): 553. """Release the semaphore 554. 555. :param tag: A tag identifying what is releasing the semaphore 556. :param acquire_token: The token returned from when the semaphore was 557. acquired. Note that this is not really needed to directly use this 558. class but is needed for API compatibility with the 559. SlidingWindowSemaphore implementation. 560. """ 561. logger.debug("Releasing acquire %s/%s" % (tag, acquire_token)) 562. self._semaphore.release() 563. 564. 565. class SlidingWindowSemaphore(TaskSemaphore): 566. """A semaphore used to coordinate sequential resource access. 567. 568. This class is similar to the stdlib BoundedSemaphore: 569. 570. * It's initialized with a count. 571. * Each call to ``acquire()`` decrements the counter. 572. * If the count is at zero, then ``acquire()`` will either block until the 573. count increases, or if ``blocking=False``, then it will raise 574. a NoResourcesAvailable exception indicating that it failed to acquire the 575. semaphore. 576. 577. The main difference is that this semaphore is used to limit 578. access to a resource that requires sequential access. For example, 579. if I want to access resource R that has 20 subresources R_0 - R_19, 580. this semaphore can also enforce that you only have a max range of 581. 10 at any given point in time. You must also specify a tag name 582. when you acquire the semaphore. The sliding window semantics apply 583. on a per tag basis. The internal count will only be incremented 584. when the minimum sequence number for a tag is released. 585. 586. """ 587. def __init__(self, count): 588. self._count = count 589. # Dict[tag, next_sequence_number]. 590. self._tag_sequences = defaultdict(int) 591. self._lowest_sequence = {} 592. self._lock = threading.Lock() 593. self._condition = threading.Condition(self._lock) 594. # Dict[tag, List[sequence_number]] 595. self._pending_release = {} 596. 597. def current_count(self): 598. with self._lock: 599. return self._count 600. 601. def acquire(self, tag, blocking=True): 602. logger.debug("Acquiring %s", tag) 603. self._condition.acquire() 604. try: 605. if self._count == 0: 606. if not blocking: 607. raise NoResourcesAvailable("Cannot acquire tag '%s'" % tag) 608. else: 609. while self._count == 0: 610. self._condition.wait() 611. # self._count is no longer zero. 612. # First, check if this is the first time we're seeing this tag. 613. sequence_number = self._tag_sequences[tag] 614. if sequence_number == 0: 615. # First time seeing the tag, so record we're at 0. 616. self._lowest_sequence[tag] = sequence_number 617. self._tag_sequences[tag] += 1 618. self._count -= 1 619. return sequence_number 620. finally: 621. self._condition.release() 622. 623. def release(self, tag, acquire_token): 624. sequence_number = acquire_token 625. logger.debug("Releasing acquire %s/%s", tag, sequence_number) 626. self._condition.acquire() 627. try: 628. if tag not in self._tag_sequences: 629. raise ValueError("Attempted to release unknown tag: %s" % tag) 630. max_sequence = self._tag_sequences[tag] 631. if self._lowest_sequence[tag] == sequence_number: 632. # We can immediately process this request and free up 633. # resources. 634. self._lowest_sequence[tag] += 1 635. self._count += 1 636. self._condition.notify() 637. queued = self._pending_release.get(tag, []) 638. while queued: 639. if self._lowest_sequence[tag] == queued[-1]: 640. queued.pop() 641. self._lowest_sequence[tag] += 1 642. self._count += 1 643. else: 644. break 645. elif self._lowest_sequence[tag] < sequence_number < max_sequence: 646. # We can't do anything right now because we're still waiting 647. # for the min sequence for the tag to be released. We have 648. # to queue this for pending release. 649. self._pending_release.setdefault( 650. tag, []).append(sequence_number) 651. self._pending_release[tag].sort(reverse=True) 652. else: 653. raise ValueError( 654. "Attempted to release unknown sequence number " 655. "%s for tag: %s" % (sequence_number, tag)) 656. finally: 657. self._condition.release() 658. 659. 660. class ChunksizeAdjuster(object): 661. def __init__(self, max_size=MAX_SINGLE_UPLOAD_SIZE, 662. min_size=MIN_UPLOAD_CHUNKSIZE, max_parts=MAX_PARTS): 663. self.max_size = max_size 664. self.min_size = min_size 665. self.max_parts = max_parts 666. 667. def adjust_chunksize(self, current_chunksize, file_size=None): 668. """Get a chunksize close to current that fits within all S3 limits. 669. 670. :type current_chunksize: int 671. :param current_chunksize: The currently configured chunksize. 672. 673. :type file_size: int or None 674. :param file_size: The size of the file to upload. This might be None 675. if the object being transferred has an unknown size. 676. 677. :returns: A valid chunksize that fits within configured limits. 678. """ 679. chunksize = current_chunksize 680. if file_size is not None: 681. chunksize = self._adjust_for_max_parts(chunksize, file_size) 682. return self._adjust_for_chunksize_limits(chunksize) 683. 684. def _adjust_for_chunksize_limits(self, current_chunksize): 685. if current_chunksize > self.max_size: 686. logger.debug( 687. "Chunksize greater than maximum chunksize. " 688. "Setting to %s from %s." % (self.max_size, current_chunksize)) 689. return self.max_size 690. elif current_chunksize < self.min_size: 691. logger.debug( 692. "Chunksize less than minimum chunksize. " 693. "Setting to %s from %s." % (self.min_size, current_chunksize)) 694. return self.min_size 695. else: 696. return current_chunksize 697. 698. def _adjust_for_max_parts(self, current_chunksize, file_size): 699. chunksize = current_chunksize 700. num_parts = int(math.ceil(file_size / float(chunksize))) 701. 702. while num_parts > self.max_parts: 703. chunksize *= 2 704. num_parts = int(math.ceil(file_size / float(chunksize))) 705. 706. if chunksize != current_chunksize: 707. logger.debug( 708. "Chunksize would result in the number of parts exceeding the " 709. "maximum. Setting to %s from %s." % 710. (chunksize, current_chunksize)) 711. 712. return chunksize


Team organising Sports fest of Army Institute of Technology
Sports secretary
Cricket secretary
Cricket secretary Basketball secretary
Atheletics secretary
Badminton secretary
Badminton secreatary
Basketball secreatary
Shaurya Khurana
Savvi Repaswal
Football Secreatary
Lawn Tennis
Squash Secreatary
Table Tenis Secreatary
Volley ball
Kabadi


AdaBoost
the main idea
- focus more on hard-to-detect objects in an image
- use a bunch of weak classifiers and train them sequentially, with each classifier focusing on areas missed by the previous weak classifier, finally aggregating them to get a strong classifier
- short for Adaptive Boosting
- not really in use for computer vision anymore - deep learning CV > traditional CV
- unsure of the term Pri ~ Dt [...]
- X - a set of training data
- m - number of training data
- yi - actual output of training datum xi
- D - weights for training data
- Dt - weights for training data during the tth training iteration
- D1(i) - weights for training datum i during the 1st training iteration
- ht - chosen weak classifier at the tth training iteration
- Pr - Probability
- εt - weighted error of the outputs of the chosen weak classifier ht at the tth training iteration
- αt - weight of the chosen classifier at the tth training iteration
- Zt - a normalisation factor at the tth training iteration, chosen such that Dt+1 is a distribution
- H - strong classifier
- given: (x1, y1), ..., (xm, ym) where xi ∈ X, yi ∈ {-1, 1}
-
for i = 1, ..., m:
- D1(i) = 1⁄m
-
for t = 1, ..., T:
- train weak learner using distribution D
- pick ht: X → {-1, 1} which minimises εt = Pri ~ Dt [ht(xi) ≠ yi]
- get αt = 1⁄2 ln(1 - εt ⁄ εt)
-
for i = 1, ..., m:
- Dt+1(i) = (Dt(i) ⁄ Zt)·e-αt·ht(xi)·yi
- output H(x) = sign_of(∑t=1, ..., T(αt·ht(x)))
- Boosting and AdaBoost clearly explained
- Adaboost for Dummies: Breaking Down the Math (and its Equations) into Simple Terms
- Boosting algorithm: AdaBoost
Focal Loss
the main idea
-
focus more on hard-to-detect objects in an image
- just like AdaBoost (different implementation though)
- TODO
- at γ = 0, FL(pt) = CE(pt)
- TODO
- y ∈ {±1} - groundtruth class: 1 for object detected, -1 for no object detected
- p ∈ [0,1] - model's estimated probability for the class with label y = 1
- CE(p, y) - cross entropy loss for binary classification: -log(p) if y = 1, -log(1-p) otherwise
- pt - same as p, but rewritten for convenience: p if y = 1, (1-p) otherwise
- CE(pt) - cross entropy loss for binary classification, rewritten for convenience: CE(p, y) = -log(pt)
- γ ≥ 0 - tunable focusing parameter that smoothly adjusts the rate at which easy examples are down-weighted
- (1 - pt)γ - modulating factor
- α - hyperparameter that weights the losses for differentclasses/background according to their relative frequency
- original version (binary classification): FL(pt) = αt * (1 - pt)γ * CE(pt)
- Focal Loss for Dense Object Detection
- Focal Loss (summary in Chinese)
- Neural Networks Intuitions: 3. Focal Loss for Dense Object Detection — Paper Explanation
- The intuition behind RetinaNet
- Review: RetinaNet — Focal Loss (Object Detection)
- Multi-class classification with focal loss for imbalanced datasets (multi-class instead of binary; has code)
- About Alpha parameter in focal loss


---
layout: talk_reveal3
title: Influenza evolutionary dynamics and global circulation patterns
venue: Leavenworth, WA
event: HIV Dynamics and Evolution
author: Trevor Bedford
---
Trevor Bedford (@trvrb)
Project to provide a real-time view of the evolving influenza population Project to provide a real-time view of the evolving influenza population All in collaboration with Richard Neher
with
Richard Neher,
James Hadfield,
Colin Megill,
Clade frequencies $X$ derive from the fitnesses $f$ and frequencies $x$ of constituent viruses, such that $$\hat{X}_v(t+\Delta t) = \sum_{i:v} x_i(t) \, \mathrm{exp}(f_i \, \Delta t)$$ This captures clonal interference between competing lineages where the fitness $f$ of virus $i$ is estimated as $$\hat{f}_i = \beta^\mathrm{HI} \, f_i^\mathrm{HI} + \beta^\mathrm{freq} \, f_i^\mathrm{freq}$$ where $f_i^\mathrm{HI}$ measures antigenic drift via HI and $f_i^\mathrm{freq}$ measures clade growth/decline We learn coefficients and validate model based on previous 15 H3N2 seasons{{ page.title }}
{{ page.date | date: "%-d %b %Y" }}
{{ page.event }}
{{ page.venue }}
HIV vs influenza
HIV within-host phylogeny vs influenza between-host phylogeny
Fundamentally driven by the same process
Antigenic drift driven by acquired immunity
Selection visible in diversity and divergence
Systematic divergence at gp120 through time
Within-host HIV shows adaptive evolution in envelope and purifying selection in enzymes
Between-host influenza shows adaptive evolution in surface proteins and purifying selection in internal proteins
Influenza evolution
Population turnover is extremely rapid
Clades emerge, die out and take over
Clades show rapid turnover
Dynamics driven by antigenic drift
Drift necessitates vaccine updates
H3N2 vaccine updates occur every ~2 years
Vaccine strain selection by WHO
Problem of applied evolutionary biology
Every paper in the field...
Disconnect between evolutionary studies and information needed by WHO
Microbial evolution workshop at the Kavli Institute for Theoretical Physics
Decided to tackle this head on and build something that
Nextflu
Nextflu
Nextflu
pipeline
Subsequently wrapped into larger efforts to do real-time analysis across pathogens with the
Nextstrain
project
Sidney Bell,
John Huddleston,
Barney Potter,
Charlton Callender,
Emma Hodcroft
Up-to-date analysis publicly available at:
nextstrain.org/flu
Current H3N2 diversity
Current H3N2 diversity
Two clades have been growing rapidly
Clade A2 more recently increasing
Reassortment event appears to drive success of clade A2
Reassortment event appears to drive success of clade A2
Antigenic analysis
Influenza hemagglutination inhibition (HI) assay
HI measures cross-reactivity across viruses
Data in the form of table of maximum inhibitory titers
Antigenic cartography compresses HI measurements into an interpretable diagram
Evolutionary model of HI titer data in which particular substitutions drop titer
Model can be used to interpolate across tree and predict phenotype of untested viruses
Model is highly predictive of missing titer values
Incorporate HI data from WHO Collaborating Centers
Recent H3N2 antigenic evolution
Forecasting
"The future is here, it's just not evenly distributed yet"
— William GibsonUSA music industry, 2011 dollars per capita
Influenza population turnover
Vaccine strain selection timeline
Seek to explain change in clade frequencies over 1 year
Fitness models can project clade frequencies
The question of forecasting becomes: how do we accurately estimate fitnesses of circulating viruses?
Fortunately, there's lots of training data and previously successful strains have had:
We predict fitness based on a simple formula
Clade growth rate is well predicted (ρ = 0.66)
Growth vs decline correct in 84% of cases
Trajectories show more detailed congruence
Trajectories show more detailed congruence
Current forecast favors drifted A1b clade
Unlikely to replace human intuition, but could automate serum and VCV selection
Geographic circulation
Geographic circulation patterns drive evolutionary outcomes
Clades commonly show region-specific circulation patterns
Phylogeny of H3 with geographic history
Infer geographic transition matrix
Geographic location of phylogeny trunk
Region-specific ancestry
Which regions are most predictive of future outcomes?
Correlation between clade frequencies
Correlations across regions
Correlations across regions with 6-month lag
Correlations across regions with 12-month lag
Next steps
Bedford Lab:
Alli Black,
Sidney Bell,
Gytis Dudas,
John Huddleston,
Barney Potter,
James Hadfield,
Louise Moncla
Influenza: WHO Global Influenza Surveillance Network, GISAID, Richard Neher, Barney Potter, John Huddleston, James Hadfield, Colin Russell, Andrew Rambaut, Dave Wentworth, Becky Garten, Jackie Katz, Marta Łuksza, Michael Lässig, Richard Reeve
Nextstrain: Richard Neher, James Hadfield, Colin Megill, Sidney Bell, Charlton Callender, Barney Potter, John Huddleston, Emma Hodcroft


Saudi Arabia key to world’s fastest-growing hotel chain
Shah Rukh Khan turned 54 on Nov. 2, and naturally, the Bollywood star’s social media was flooded with well-wishes from friends and fans. However, one of the most touching came from the world’s tallest building. In celebration of the actor’s birthday on Saturday night, the Burj Khalifa lit up with a special tribute to Khan. The spectacular projection included a thoughtful message that read “Happy birthday to the King of Bollywood” as hundreds gathered for the Dubai Fountain show. The sweet gesture was not lost on the “Om Shanti Om” actor, who took to his Twitter to thank Mohamed Alabbar, founder of Emaar Properties, and the iconic landmark. “Your love and kindness is unsurpassable. Wow! This is really the tallest I have ever been. Love you Dubai,” he wrote alongside a clip of the light show.


|
Bolt Preview
0.4.4
C++ template library with support for OpenCL
|
Yes, Bolt includes a small static library that must be linked into a user program to properly resolve symbols. Since Bolt is a template library, almost all functionality is written in header files, but OpenCL follows an online compilation model to be included in a library, not in header files.
Yes, Bolt contains dependencies on Boost. All dependent header files and pre-compiled libraries are available in the Bolt SDK.
Transform, reduce, transform_reduce, count, count_if, inclusive_scan, exclusive_scan, and sort.
Bolt uses features available in OpenCL v1.2.
Yes, Bolt requires the use of templates in kernel code; currently, AMD is the only vendor to provide this support.
Generally speaking, the latest Catalyst package contains the most recent OpenCL runtime. As of this writing, the recommended Catalyst package is 12.10.
Bolt is planned to be completely open source in the near future. Most of the algorithm functionality is located in header files, so a majority of the source is already available.
The Apache License, Version 2.0.
bolt::cl::device_vector manages device-local memory and can deliver higher performance on discrete GPU system; however, host memory interfaces eliminate the need to create and manage device_vectors. If memory is re-used across multiple Bolt calls, or is referenced by other kernels, using device_vector delivers higher performance.


| MATLAB 文件帮助: cv.GeneralizedHoughBallard/detect | 查看 cv.GeneralizedHoughBallard/detect 的代码 | Index |
detect Find template on image
[positions,votes] = hough.detect(image)
[positions,votes] = hough.detect(edges, dx, dy)
edges
and
single-precision floating type.dx
.[posx, posy, scale, angle]
positions
.| Access | public |
| Sealed | false |
| Static | false |


Sylvanian Families Bramble Hedgehog Baby (Pink)
The adorable Bramble family comes with combs and brushes.
UK Biography Information
Baby Sister Peaches Bramble is a very, very naughty little girl! She's alway taking her father's tailor's chalk and drawing patterns on the walls and furniture.


Nick Saban Reacts To Mac Jones’ Performance vs. Western Carolina
by Alek ArendTua Tagovailoa’s unfortunate season-ending injury last week against Mississippi State has opened the door for Alabama QB Mac Jones. The quarterback got his first start under center on Saturday against Western Carolina.
The Crimson Tide QB was nearly perfect, going 10-of-12 for 275 yards and three touchdowns. It was certainly an impressive start for the young quarterback.
Alabama head coach Nick Saban discussed Jones’ performance after the Crimson Tide’s 66-3 victory on Saturday. Saban kept his comments brief.
“Mac was efficient.” Saban said, via Mike Rodak of AL.com. “Made a lot of explosive plays. Maybe didn’t throw it as much.”
Jones and the Crimson Tide are trying to do all they can to impress the College Football Playoff committee. Today’s victory won’t move the needle in regards to where Alabama should be ranked. But next’s week Iron Bowl will be pivotal.
The Crimson Tide have one last shot to try and convince the committee they should be one of the four teams in the playoff. Many experts feel because of Tagovailoa’s injury, the Tide don’t deserve a top-four ranking. But Jones will certainly try and prove the committee wrong.
If Alabama can come away with a convincing win over Auburn next week, that might be enough to earn a spot in the CFP.
Alabama-Auburn kicks off next Saturday at 3:30 p.m. ET on CBS.


This site is best experienced with the sound turned on
REMIX! Switch to portrait mode to best experience our site.
Just Tom is Tom Thorney and friends.
Our team believes in providing the best audio experience, no matter what the project, so we put our website on vinyl. You can’t hear it right now, but it’s pretty great. Click listen on the right to get the full experience.
↓
Services
↓
Stock Library
We have one of the most extensive stock music libraries in North America, with thousands of tracks sourced through an unrivaled worldwide network of expert composers.
Original Music
We know Shift+4 is king. By paying upfront, we get the world’s best composers and songwriters to help bring your ideas to life.
Sound Design
Our extensive work in film and TV has us aligned with the top sound designers in North America. Hit listen on the right to hear what we can do.
Music Licensing
We’ve placed thousands of tracks for television shows, movies, and advertising, and have long-standing relationships with publishing companies and record labels.
Voice Directing
Tom’s been directing celebrities, musicians, and voice over actors, union and non-union, for almost 30 years.
Tom
Musicians, actors, writers, producers, and clients love Tom’s low-key control. Having spent time in each role, he understands their unique needs. If you were listening instead of reading, you’d be hearing his smooth voice.
At Just Tom, we have over 30 years of experience crafting radio, TV, and more for a variety of clients.
Our combination of Canada’s best stock music library, and a worldwide network of accomplished composers and songwriters, means we create Canada’s best audio - period.
Use the buttons below to explore some of our latest work.


Sidebery
Invoked action require some additional permissions:
Request 'Websites data' permission
This can be undone in sidebery settings section 'permissions'.


{% load static %}
The Engineering Placement Quiz was built by a group of five Management Engineering students for their Fourth Year Design Project.
The quiz is powered by a Naive Bayes classifier algorithm - one of the most efficient, intuitive and effective algorithms for applying machine learning to large data sets.
The Naive Bayes algorithm leverages Bayes’ Theorem to calculate the probability of a sample belonging to a certain category. Speaking in terms of The Quiz, it is calculating the probability that you (the sample) should belong to each of the fifteen engineering programs offered at the University of Waterloo (the categories). It is calculating these probabilities based on the ~2000 data points it has been trained with.
In machine learning we are often interested in selecting the best hypothesis (H) given a set of data (D). Bayes’ Theorem provides a way to calculate the probability of a hypothesis given the available data.
Bayes’ Theorem Equation: P(H | D) =P(D | H) x P(H)P(D)
Where:
P(H | D) is the probability of hypothesis H given the data D. This is called the posterior probability.
P(D | H) is the probability of data D given that the hypothesis H was true.
P(H) is the probability of hypothesis H being true (regardless of the data). This is called the prior probability of H.
P(D) is the probability of the data (regardless of the hypothesis).
Let’s say we have data on 1000 pieces of fruit. The fruit being a lemon, mango or some other fruit. Imagine we also know 3 features of each fruit - whether it’s sour or not, round or not and yellow or not. We’ve organized all this data in the table below.
|
Fruit |
Sour |
Round |
Yellow |
Total |
|
Lemon |
400 |
350 |
450 |
500 |
|
Mango |
0 |
150 |
300 |
300 |
|
Other |
100 |
150 |
50 |
200 |
|
Total |
500 |
650 |
800 |
1000 |
Just from looking at the table, we already know that:
50% of the total fruits are lemons
30% are mangos
20% are other fruits
Of the 500 lemons, 400 (80%) are sour, 350 (70%) are round and 450 (90%) are yellow
Of the 300 mangos, 0 are sour, 150 (50%) are round and 300 (100%) are yellow
Of the other 200 fruits, 100 (50%) are sour, 150 (75%) are round and 50 (25%) are yellow
Let’s say we’re given the features of an additional piece of fruit and we want to predict what type of fruit it is (it’s class). We’re told that the fruit is sour, round, and yellow. We can use Bayes’ Theorem to classify whether it’s a lemon, a mango or other fruit.
General Formula: P(A | B) =P(B | A) x P(A)P(B)
Lemon:P(Lemon | Sour, Round, Yellow) =P(Sour | Lemon) x P(Round | Lemon) x P(Yellow | Lemon) x P(Lemon)P(Sour) x P(Round) x P(Yellow)
P(Lemon | Sour, Round, Yellow) =(0.8) x (0.7) x (0.9) x (0.5)(0.25) x (0.33) x (0.41)
P(Lemon | Sour, Round, Yellow) =0.252
Mango: P(Mango | Sour, Round, Yellow)=P(Sour| Mango) x P(Round | Mango) x P(Yellow | Mango) x P(Mango)P(Sour) x P(Round) x P(Yellow)
P(Mango | Sour, Round, Yellow)=0
Other: P(Other | Sour, Round, Yellow) =P(Sour | Other) x P(Round | Other) x P(Yellow | Other) x P(Other)P(Sour) x P(Round) x P(Yellow)
P(Other | Sour, Round, Yellow) =0.018
Therefore, based on the highest score (~25.2% for lemon) we can assume this sour, round and yellow fruit is in fact, a lemon.


Elle is training to become a full-stack developer in Cohort 6 at Ada Developers Academy in Seattle, WA. She is interested in technology, social justice, and how those two fields can be friends.
Elle has lived in the American Southwest and the Pacific Northwest. Her background is in Printmaking and foreign languages.


class: center, middle # Introducing Spring Cloud -- Introductory level --- # WIP Introduction WIP - Cloud Native? Cloud Ready? WTF? - 12 Factors - Infra to support: TA DA! Spring-Cloud - Netfilx OSS - no bs. battle hardened. --- # Cloud Native? -- # Cloud Ready? -- #Software as a Service? --- background-image: url(image/cloud-salad.png) # WTF? --- background-image: url(image/12-factor.png) --- # [12factor.net](12factor.net) -- - A methodology for building software-as-a-service apps -- - Defines being _cloud ready_ or _cloud native_ -- - __Do__ go and look over the 12 factors --- # Enabled by Spring Boot Project -- ## We'll come back to this... --- # "I just want to run my code" --- ## You'll need... -- - Service registration and discovery -- - Routing -- - Load balancing -- - Circuit Breakers -- - Lot's of _STUFF_ (hard to get right) -- - the TIME to get it right (and fix it when you don't) --- # Don't Write the Nasty Part of the Sandwich --- # Spring Cloud -- ## Out of the Box Usability -- ## Abstractions of Common Cloud Use-Cases -- - Service Discovery, for example -- ## Crazy Simple To Enable Spring Boot Apps --- background-image: url(image/netflix-oss.png) # Battle Hardened No BS Cloud Components --- # Delivered by Spring Cloud Netflix -- - Netflix OSS integrations for Spring Boot -- - Annotations -- - Autoconfigurations -- - Spring model idioms -- - _SIMPLE!_ --- # Back up and start at the bottom --- # Spring Boot -- - No Fuss -- - Twelve-Factor Suitable -- - Opinionated* -- .footnote[*but, quickly gets out of the way as needed] --- ## Spring CLI -- - fastest way to get started with Spring Boot -- - twelve-factor friendly packaging -- - testing -- - encourages best practices -- - rapid development --- class: center, middle # _Radically Fast_ --- ### Put this in your pipe single file named: _app.groovy_ ```java @RestController class PipeStuff { @RequestMapping("/") String home() { "Hello World!" } } ``` --- ### And Smoke It .small[ ```terminal kbowen@kenneth-devbox64:/tmp$ spring run app.groovy Resolving dependencies............. . ____ _ __ _ _ /\\ / ___'_ __ _ _(_)_ __ __ _ \ \ \ \ ( ( )\___ | '_ | '_| | '_ \/ _` | \ \ \ \ \\/ ___)| |_)| | | | | || (_| | ) ) ) ) ' |____| .__|_| |_|_| |_\__, | / / / / =========|_|==============|___/=/_/_/_/ :: Spring Boot :: (v1.2.5.RELEASE) 2015-09-15 10:08:17.812 INFO 17579 --- [ runner-0] o.s.boot.SpringApplication : Starting application on kenneth-devbox64 with PID 17579 (/home/kbowen/.m2/repository/org/springframework/boot/spring-boot/1.2.5.RELEASE/spring-boot-1.2.5.RELEASE.jar started by kbowen in /tmp) ... 2015-09-15 10:08:30.486 INFO 17579 --- [ runner-0] o.s.boot.SpringApplication : Started application in 13.511 seconds (JVM running for 32.411) ``` ] -- ![Default-aligned hello](image/hello_world.png) --- # Bootstrapping App Development --- Impressive Number of Archetypes .small[```terminal kbowen@kenneth-devbox64:~$ spring init --list +-----------------------------------+--------------------------------------------------------------------------------------------------+ | Id | Description | +-----------------------------------+--------------------------------------------------------------------------------------------------+ | actuator | Production ready features to help you monitor and manage your application | | actuator-docs | API documentation for the Actuator endpoints | | amqp | Support for the Advanced Message Queuing Protocol via spring-rabbit | | aop | Support for aspect-oriented programming including spring-aop and AspectJ | | batch | Support for Spring Batch including HSQLDB database | | cache | Support for Spring's Cache abstraction | | cloud-aws | Support for spring-cloud-aws | | cloud-aws-jdbc | Support for spring-cloud-aws-jdbc | | cloud-aws-messaging | Support for spring-cloud-aws-messaging | | cloud-bus-amqp | Support for spring-cloud-bus-amqp | | cloud-cloudfoundry-discovery | Support for spring-cloud-cloudfoundry-discovery (service discovery with Cloud Foundry) | | cloud-cluster-hazelcast | Support for spring-cloud-cluster-hazelcast (leadership election and global state with Hazelcast) | | cloud-cluster-redis | Support for spring-cloud-cluster-redis (leadership election and global state with Redis) | | cloud-cluster-zookeeper | Support for spring-cloud-cluster-zookeeper (leadership election and global state with Zookeeper) | | cloud-config-client | Support for spring-cloud-config Client | | cloud-config-server | Support for spring-cloud-config Server | | cloud-connectors | Simplifies connecting to services in cloud platforms | | cloud-eureka | Support for spring-cloud-netflix Eureka | | cloud-eureka-server | Support for spring-cloud-netflix Eureka Server | | cloud-feign | Support for spring-cloud-netflix Feign | | cloud-hystrix | Support for spring-cloud-netflix Hystrix | | cloud-hystrix-dashboard | Support for spring-cloud-netflix Hystrix Dashboard | | cloud-oauth2 | Support for spring-cloud-security OAuth2 | | cloud-ribbon | Support for spring-cloud-netflix Ribbon | | cloud-security | Support for spring-cloud-security | | cloud-starter | Support for spring-cloud-context (e.g. Bootstrap context and @RefreshScope) | ... ```] -- __80 Archetypes in all!__ .small[(today)] --- background-image: url(image/spring_start.png) --- class: center, middle # Let's Build an App


Instructions: enter line-separated names (in lastname, firstname format) in the textbox.Remove people who aren't walking or type NW next to them. Press Try it and the seating will be ready.


--- layout: page bigimg: - "/img/logo-osr2020_cropped.png" ---
The OS-SIG Open Science Room (OSR) is a conference within a conference. It is part of the Organisation for Human Brain Mapping (OHBM) annual meeting.
The OSR aims to be a welcoming and inclusive space for discussion around open practices within neuroimaging and science more generally. Joining in with the activities of the OSR is a key opportunity to connect with others in the community, learn from each other, and start collaborations to build a more inclusive, transparent and future-ready scientific field.
This year the OHBM meeting and OSR will be open for virtual attendance only!
The OHBM meeting runs from Tuesday 23rd June 2020 to Friday 3rd July 2020, for two weeks instead of the usual five days, to give all virtual attendees full opportunity to soak up the experience without being glued to your screens for nine hours per day!
The OSR kicks off with a welcome party on Monday 22nd June 2020 and runs in parallel to the OHBM program, with almost no overlaps between the two schedules.
For increased global accessibility, the daily OSR schedule repeat three times per 24 hours, once per time zone hub.
The OHBM meeting will be hosted on an interactive virtual platform, with opportunities for networking alongside the standard keynotes, symposia, exhibitor booths, and poster sessions.
OHBM registrants will be able to access the OS-SIG's Open Science Room via this platform, alongside the rest of the OHBM meeting content.
To reduce barriers to access, the OSR talks, discussions and interactions this year will also be available for free accross all timezones, serving our global community.
OHBM registrants will have full access to the OSR via the virtual conferencing platform. If you are registered to attend OHBM, you do not have to register additionally via the OSR website. Zero cost participants can register via the OSR website to gain free access to OSR content and interactions.
The OSR hosts content in various formats: short lightning talks, longer 'keynote' talks, software demonstrations, and emergent discussions, all centred around themes relevant to open science practices in our discipline. Emergent sessions can be booked now or during the meeting itself via the submit page.
Talks, discussions and demonstrations in the OSR will be fresh, relevant and challenging. We aim to address new and emerging topics, fields and concerns, along with big picture issues in open science. The OSR is also a space to meet, hang out, work and collaborate! We aim to create virtual group working and interaction spaces, to support you in making real connections with other virtual attendees.
The OSR is a space for everyone. We come with diverse skill sets, experiences and viewpoints, and we can all learn something new! We expect visitors to the OSR to comply with the Code of Conduct for the OHBM annual meeting. These behaviours are aligned with open science practices of sharing, collaboration, and inclusivity. In the OSR we aim to model an academic community we wish to be a part of, where we interact with each other with curiosity, kindness and respect.
To join the OSR, register now.
Registration is open for the duration of the event, and is required for zero-cost attendance.
After registration, you will receive the information necessary to access OSR talks and discussions.
Note: separate OSR registration is not required for those already registered for the OHBM meeting, as they will receive the necessary information to connect to the OSR via the virtual conferencing platform.
Participation in the OSR will happen across several platforms. OHBM registrants can view OSR content and interact with the community from within the virtual conferencing platform. Both OHBM and Zero-cost participants will be able to join the interactive program via Crowdcast. Crowdcast allows us to stream all our pre-recorded talks, have live question-and-answer sessions with speakers, and interact via text and video with our global community. We'll additionally make a text based Mattermost environment available for chats with speakers and each other, and asking our Open Science experts and staff for help. For informal social activities, a core part of a scientific conference, we'll use Gather.town, which was a great hit at the recent virtual OHBM Brainhack.
We would be very pleased to hear from you to discuss any aspect of the OSR! Our communication channels are listed here.
Images throughout this site are included with permission from Undraw, The Noun Project, and OSR Volunteers.


Storage cache
get data
get data (expire 10s)
get data (RND argument and expire 10s)
cls
clear localStorage
clear cache 1
clear cache 2
clear cache 3


Web Developer by day, Redditor by night and Browncoat for life. Geek. Origamist, FOSS Supporter, Avid Reader, Pythonista.
Currently hacking on RoR at www.yournextleap.com


module BatFingerTree:{
..}
The finger tree itself is polymorphic over the measure and the measurement function (this is needed because sometimes the type of the measure depends on the type of the elements).
This module also contains an instanciation of a finger tree that implements a functional sequence with the following characteristics:
(something, _, _) wrap
, just pretend it is simply
the type something
(this is what the documentation does).
Complexities are given assuming that the monoid combination operation and the measurement functions are constant time and space.
None of the functions on finger trees can cause stack overflow:
they use at worst a logarithmic amount of stack space.
type monoid 'a
= {
|
zero : |
(* |
The neutral element of the monoid.
| *) |
|
combine : |
(* | combine should be associative, and have zero as neutral element. | *) |
exception Empty
module type S ={
..}
module Generic:{
..}
type t 'a
include BatFingerTree.S
let size : t 'a => int
size t
returns the number of elements in the sequence.
Unlike the generic size
on finger trees, this one has complexity O(1).
let split_at : t 'a => int => (t 'a, t 'a)
split_at
is equivalent to List.split_at
.Invalid_argument
when the index is out of bounds.
O(log(n)).
let get : t 'a => int => 'a
get t i
returns the i
-th element of t
.Invalid_argument
when the index is out of bounds.
O(log(n)).
let set : t 'a => int => 'a => t 'a
set t i v
returns t
where the i
-th element is now v
.Invalid_argument
when the index is out of bounds.
O(log(n)).
let update : t 'a => int => ('a => 'a) => t 'a
update t i f
returns t
where the i
-th element is now f (get i t)
.Invalid_argument
when the index is out of bounds.
O(log(n)).


My Queries
Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldu
22 january 2019, 20 minutes ago
TitLorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum
Sent By:64059


It is legal to do this ― but the text will appear in the
DOS window where println
has always put it,
not in the frame where you want it.
drawString()
methodOrdinarily a child class inherits every method defined in its parent class. A child overrides a method in its parent by defining a new method with the same signature. Now the new method will be used in place of the parent's method in a child object.
When the Java system needs to paint a myFrame
object
it first does most of the work
and then calls paint()
in the myFrame
object to do whatever
extra the programmer wants.
The first part of the work is vital.
It involves interacting with the graphics
of the computer system and controling the graphics board.
After all of that you get to ask for the little extra bit that
you are interested in.
Our paint()
looks like this:
public void paint ( Graphics g ) { // draw a String at location x=10 y=50 g.drawString("A myFrame object", 10, 50 ); }
The parameter g
is a reference to a Graphics
object.
The Graphics
object represents the part of the frame that you
can draw on.
When the system calls paint()
,
the system gives it this parameter.
One of its methods is
drawString(String st, int X, int Y)
Here is MyFrame
again, with a few changes.
class myFrame extends Frame { public void paint ( Graphics g ) { g.drawString(________, __________, ________); } }


Hello,
You can change your account page by clicking "Reset password" button, this link will expire in 1 hours.
Reset password
Thanks
{{ siteTitle }} {{ domain }}


New in version 0.3.
Applications need some kind of configuration. There are different settings you might want to change depending on the application environment like toggling the debug mode, setting the secret key, and other such environment-specific things.
The way Flask is designed usually requires the configuration to be available when the application starts up. You can hardcode the configuration in the code, which for many small applications is not actually that bad, but there are better ways.
Independent of how you load your config, there is a config object available which holds the loaded configuration values: The config attribute of the Flask object. This is the place where Flask itself puts certain configuration values and also where extensions can put their configuration values. But this is also where you can have your own configuration.
The config is actually a subclass of a dictionary and can be modified just like any dictionary:
app = Flask(__name__)
app.config['DEBUG'] = True
Certain configuration values are also forwarded to the Flask object so you can read and write them from there:
app.debug = True
To update multiple keys at once you can use the dict.update() method:
app.config.update(
DEBUG=True,
SECRET_KEY='...'
)
The following configuration values are used internally by Flask:
| DEBUG | enable/disable debug mode |
| TESTING | enable/disable testing mode |
| PROPAGATE_EXCEPTIONS | explicitly enable or disable the propagation of exceptions. If not set or explicitly set to None this is implicitly true if either TESTING or DEBUG is true. |
| PRESERVE_CONTEXT_ON_EXCEPTION | By default if the application is in debug mode the request context is not popped on exceptions to enable debuggers to introspect the data. This can be disabled by this key. You can also use this setting to force-enable it for non debug execution which might be useful to debug production applications (but also very risky). |
| SECRET_KEY | the secret key |
| SESSION_COOKIE_NAME | the name of the session cookie |
| SESSION_COOKIE_DOMAIN | the domain for the session cookie. If this is not set, the cookie will be valid for all subdomains of SERVER_NAME. |
| SESSION_COOKIE_PATH | the path for the session cookie. If this is not set the cookie will be valid for all of APPLICATION_ROOT or if that is not set for '/'. |
| SESSION_COOKIE_HTTPONLY | controls if the cookie should be set with the httponly flag. Defaults to True. |
| SESSION_COOKIE_SECURE | controls if the cookie should be set with the secure flag. Defaults to False. |
| PERMANENT_SESSION_LIFETIME | the lifetime of a permanent session as datetime.timedelta object. Starting with Flask 0.8 this can also be an integer representing seconds. |
| USE_X_SENDFILE | enable/disable x-sendfile |
| LOGGER_NAME | the name of the logger |
| SERVER_NAME | the name and port number of the server. Required for subdomain support (e.g.: 'myapp.dev:5000') Note that localhost does not support subdomains so setting this to “localhost” does not help. Setting a SERVER_NAME also by default enables URL generation without a request context but with an application context. |
| APPLICATION_ROOT | If the application does not occupy a whole domain or subdomain this can be set to the path where the application is configured to live. This is for session cookie as path value. If domains are used, this should be None. |
| MAX_CONTENT_LENGTH | If set to a value in bytes, Flask will reject incoming requests with a content length greater than this by returning a 413 status code. |
| SEND_FILE_MAX_AGE_DEFAULT: | Default cache control max age to use with send_static_file() (the default static file handler) and send_file(), in seconds. Override this value on a per-file basis using the get_send_file_max_age() hook on Flask or Blueprint, respectively. Defaults to 43200 (12 hours). |
| TRAP_HTTP_EXCEPTIONS | If this is set to True Flask will not execute the error handlers of HTTP exceptions but instead treat the exception like any other and bubble it through the exception stack. This is helpful for hairy debugging situations where you have to find out where an HTTP exception is coming from. |
| TRAP_BAD_REQUEST_ERRORS | Werkzeug’s internal data structures that deal with request specific data will raise special key errors that are also bad request exceptions. Likewise many operations can implicitly fail with a BadRequest exception for consistency. Since it’s nice for debugging to know why exactly it failed this flag can be used to debug those situations. If this config is set to True you will get a regular traceback instead. |
| PREFERRED_URL_SCHEME | The URL scheme that should be used for URL generation if no URL scheme is available. This defaults to http. |
More on SERVER_NAME
The SERVER_NAME key is used for the subdomain support. Because Flask cannot guess the subdomain part without the knowledge of the actual server name, this is required if you want to work with subdomains. This is also used for the session cookie.
Please keep in mind that not only Flask has the problem of not knowing what subdomains are, your web browser does as well. Most modern web browsers will not allow cross-subdomain cookies to be set on a server name without dots in it. So if your server name is 'localhost' you will not be able to set a cookie for 'localhost' and every subdomain of it. Please chose a different server name in that case, like 'myapplication.local' and add this name + the subdomains you want to use into your host config or setup a local bind.
New in version 0.4: LOGGER_NAME
New in version 0.5: SERVER_NAME
New in version 0.6: MAX_CONTENT_LENGTH
New in version 0.7: PROPAGATE_EXCEPTIONS, PRESERVE_CONTEXT_ON_EXCEPTION
New in version 0.8: TRAP_BAD_REQUEST_ERRORS, TRAP_HTTP_EXCEPTIONS, APPLICATION_ROOT, SESSION_COOKIE_DOMAIN, SESSION_COOKIE_PATH, SESSION_COOKIE_HTTPONLY, SESSION_COOKIE_SECURE
New in version 0.9: PREFERRED_URL_SCHEME
Configuration becomes more useful if you can store it in a separate file, ideally located outside the actual application package. This makes packaging and distributing your application possible via various package handling tools (Deploying with Distribute) and finally modifying the configuration file afterwards.
So a common pattern is this:
app = Flask(__name__)
app.config.from_object('yourapplication.default_settings')
app.config.from_envvar('YOURAPPLICATION_SETTINGS')
This first loads the configuration from the yourapplication.default_settings module and then overrides the values with the contents of the file the YOURAPPLICATION_SETTINGS environment variable points to. This environment variable can be set on Linux or OS X with the export command in the shell before starting the server:
$ export YOURAPPLICATION_SETTINGS=/path/to/settings.cfg
$ python run-app.py
* Running on http://<IP_ADDRESS>:5000/
* Restarting with reloader...
On Windows systems use the set builtin instead:
>set YOURAPPLICATION_SETTINGS=\path\to\settings.cfg
The configuration files themselves are actual Python files. Only values in uppercase are actually stored in the config object later on. So make sure to use uppercase letters for your config keys.
Here is an example of a configuration file:
# Example configuration
DEBUG = False
SECRET_KEY = '?\xbf,\xb4\x8d\xa3"<\x9c\xb0@\x0f5\xab,w\xee\x8d$0\x13\x8b83'
Make sure to load the configuration very early on, so that extensions have the ability to access the configuration when starting up. There are other methods on the config object as well to load from individual files. For a complete reference, read the Config object’s documentation.
The downside with the approach mentioned earlier is that it makes testing a little harder. There is no single 100% solution for this problem in general, but there are a couple of things you can keep in mind to improve that experience:
Most applications need more than one configuration. There should be at least separate configurations for the production server and the one used during development. The easiest way to handle this is to use a default configuration that is always loaded and part of the version control, and a separate configuration that overrides the values as necessary as mentioned in the example above:
app = Flask(__name__)
app.config.from_object('yourapplication.default_settings')
app.config.from_envvar('YOURAPPLICATION_SETTINGS')
Then you just have to add a separate config.py file and export YOURAPPLICATION_SETTINGS=/path/to/config.py and you are done. However there are alternative ways as well. For example you could use imports or subclassing.
What is very popular in the Django world is to make the import explicit in the config file by adding an from yourapplication.default_settings import * to the top of the file and then overriding the changes by hand. You could also inspect an environment variable like YOURAPPLICATION_MODE and set that to production, development etc and import different hardcoded files based on that.
An interesting pattern is also to use classes and inheritance for configuration:
class Config(object):
DEBUG = False
TESTING = False
DATABASE_URI = 'sqlite://:memory:'
class ProductionConfig(Config):
DATABASE_URI = 'mysql://user@localhost/foo'
class DevelopmentConfig(Config):
DEBUG = True
class TestingConfig(Config):
TESTING = True
To enable such a config you just have to call into from_object():
app.config.from_object('configmodule.ProductionConfig')
There are many different ways and it’s up to you how you want to manage your configuration files. However here a list of good recommendations:
New in version 0.8.
Flask 0.8 introduces instance folders. Flask for a long time made it possible to refer to paths relative to the application’s folder directly (via Flask.root_path). This was also how many developers loaded configurations stored next to the application. Unfortunately however this only works well if applications are not packages in which case the root path refers to the contents of the package.
With Flask 0.8 a new attribute was introduced: Flask.instance_path. It refers to a new concept called the “instance folder”. The instance folder is designed to not be under version control and be deployment specific. It’s the perfect place to drop things that either change at runtime or configuration files.
You can either explicitly provide the path of the instance folder when creating the Flask application or you can let Flask autodetect the instance folder. For explicit configuration use the instance_path parameter:
app = Flask(__name__, instance_path='/path/to/instance/folder')
Please keep in mind that this path must be absolute when provided.
If the instance_path parameter is not provided the following default locations are used:
Uninstalled module:
/myapp.py
/instance
Uninstalled package:
/myapp
/__init__.py
/instance
Installed module or package:
$PREFIX/lib/python2.X/site-packages/myapp
$PREFIX/var/myapp-instance
$PREFIX is the prefix of your Python installation. This can be /usr or the path to your virtualenv. You can print the value of sys.prefix to see what the prefix is set to.
Since the config object provided loading of configuration files from relative filenames we made it possible to change the loading via filenames to be relative to the instance path if wanted. The behavior of relative paths in config files can be flipped between “relative to the application root” (the default) to “relative to instance folder” via the instance_relative_config switch to the application constructor:
app = Flask(__name__, instance_relative_config=True)
Here is a full example of how to configure Flask to preload the config from a module and then override the config from a file in the config folder if it exists:
app = Flask(__name__, instance_relative_config=True)
app.config.from_object('yourapplication.default_settings')
app.config.from_pyfile('application.cfg', silent=True)
The path to the instance folder can be found via the Flask.instance_path. Flask also provides a shortcut to open a file from the instance folder with Flask.open_instance_resource().
Example usage for both:
filename = os.path.join(app.instance_path, 'application.cfg')
with open(filename) as f:
config = f.read()
# or via open_instance_resource:
with app.open_instance_resource('application.cfg') as f:
config = f.read()


MISR Toolkit Python Interface
Previous:
2.9 MtkRegression
Up:
MISR Toolkit Python Interface
Next:
3.1 MtkBlockCorners
3 Containers
All the classes in this sections are returned from other functions.
Subsections
3.1 MtkBlockCorners
3.2 MtkGeoBlock
3.3 MtkGeoCoord
3.4 MtkGeoRegion
3.5 MtkSomCoord
3.6 MtkSomRegion
3.7 MtkProjParam
3.8 MtkRegCoeff
MISR Toolkit Python Interface
Previous:
2.9 MtkRegression
Up:
MISR Toolkit Python Interface
Next:
3.1 MtkBlockCorners


You have reached the American College of Veterinary Pathologists Global Health Pathology Network (GHPN) website. We are a small, but growing team across professions within pathology to serve as a centralized resource and knowledge base to facilitate Pathology Engagement in Global Health. |


Selena Gomez was cruelly trolled after she gained weight following Lupus diagnosis
Singer Selena Gomez, 27, admitted the abuse she received after she started to gain weight 'messed her up'
by Molly Pike, https://www.mirror.co.uk/authors/molly-pike/Selena Gomez has opened up about the cruel trolling she received after she gained weight following her Lupus diagnosis.
The Come and Get It singer, 27, admitted the abuse she received "messed her up" as she struggled to deal with the condition.
She has been fighting the life-threatening disease since 2013 and underwent chemotherapy and a kidney transplant to treat it.
The medication she was taking meant she put on weight, something she says she was "attacked for".
Speaking on Raquelle Stevens’ video podcast Giving Back Generation, she said: "It's the medication I have to take for the rest of my life — it depends on even the month, to be honest.
"So for me, I really noticed when people started attacking me for that. And in reality, that's just my truth.
"I fluctuate. It depends what's happening in my life.
"That got to me big time. That really messed me up for a bit."
Selena said she now doesn't open up as much on social media because of the criticism she gets.
She continued: "I don't care to expose myself to everyone and hear what they have to say about it."
Lupus is an autoimmune disease where the body's immune system becomes overly active and attacks normal, healthy tissue.
The condition is chronic and ranges from mild to severe. Lupus can affect any part of the body - so it could affect the skin, joints or internal organs.
Selena's friend Francia donated a kidney to her in 2017.
In an open Instagram update, the star shared an image of herself on a hospital bed following her treatment.


Single-Feature SelectionsSelect two features from the rank plot (left) to display their log ratios for each sample in the sample scatterplot (right).
|
Multi-Feature SelectionsUse the controls here to show more complex log ratios in the sample scatterplot. Filter numerator to features that Filter denominator to features that
|


This shows the configuration during the demo.
Start the Demo There will be a pause the first time you do this as SNMP discovery is done to find the SNMP agents on your net.


About Me
Technologist
You can find me on most places of the internet:
twitter.com/nickcalabs
linkedin.com/in/nickcalabro
github.com/nickcalabs
medium.com/@NickCalabs/
<EMAIL_ADDRESS>YouTube • Thinkful • Show-Score • Goodreads • Quora • Genius • Product Hunt • Letterboxd • Stack Exchange


{% include head.html %}
{% include navigation.html %}
{% include secondaryNavigation.html %}
{{ content }} {% include footer.html %}
{% include javascripts.html %}


Save
File Name
Open File
File Name:
File Size:
File input
Zoom Ratio
Choose a color
R:
G:
B:
#2e95ef
Choose a color
R:
G:
B:
#ffa500
Save Color To
Stroke Color
Fill Color
Shape Settings
Stroke Width
Fill
Line Settings
Line Width
Line Cap
butt
round
square
Line Join
bevel
round
milter
Miter Limit
Text Settings
Text Size
Text Font
1
2
3
4
5
Resize Image
Width
Height
Connect to server
Host Name
Open Server
Oen Server?
keyboard_arrow_down
save
folder
undo
redo
panorama_fish_eye
brightness_1
colorize
crop_din
panorama_fish_eye
remove
edit
Your device doesn't support canvas


|
||||||||||
| PREV NEXT | FRAMES NO FRAMES |
| Packages that use Writable | |
|---|---|
| org.apache.hadoop.dfs | A distributed implementation of FileSystem . |
| org.apache.hadoop.examples | Hadoop example code. |
| org.apache.hadoop.io | Generic i/o code for use when reading and writing data to the network, to databases, and to files. |
| org.apache.hadoop.ipc | Tools to help define network clients and servers. |
| org.apache.hadoop.mapred | A system for scalable, fault-tolerant, distributed computation over large data collections. |
| org.apache.hadoop.mapred.lib | Library of generally useful mappers, reducers, and partitioners. |
| org.apache.hadoop.tools | |
| org.apache.hadoop.util | Common utilities. |
| Uses of Writable in org.apache.hadoop.dfs |
|---|
| Classes in org.apache.hadoop.dfs that implement Writable | |
|---|---|
class |
DatanodeID
DatanodeID is composed of the data node name (hostname:portNumber) and the data storage ID, which it currently represents. |
class |
DatanodeInfo
DatanodeInfo represents the status of a DataNode. |
| Uses of Writable in org.apache.hadoop.examples |
|---|
| Methods in org.apache.hadoop.examples with parameters of type Writable | |
|---|---|
void |
RandomWriter.Map.map(WritableComparable key,
Writable value,
OutputCollector output,
Reporter reporter)
Given an output filename, write a bunch of random records to it. |
void |
NNBench.Map.map(WritableComparable key,
Writable value,
OutputCollector output,
Reporter reporter)
Given a number of files to create, create and open those files for both writing and reading a given number of bytes. |
void |
WordCount.MapClass.map(WritableComparable key,
Writable value,
OutputCollector output,
Reporter reporter)
|
void |
PiBenchmark.PiMapper.map(WritableComparable key,
Writable val,
OutputCollector out,
Reporter reporter)
Map method. |
| Uses of Writable in org.apache.hadoop.io |
|---|
| Subinterfaces of Writable in org.apache.hadoop.io | |
|---|---|
interface |
WritableComparable
An interface which extends both Writable and Comparable . |
| Classes in org.apache.hadoop.io that implement Writable | |
|---|---|
class |
ArrayWritable
A Writable for arrays containing instances of a class. |
class |
BooleanWritable
A WritableComparable for booleans. |
class |
BytesWritable
A byte sequence that is usable as a key or value. |
class |
CompressedWritable
A base-class for Writables which store themselves compressed and lazily inflate on field access. |
class |
FloatWritable
A WritableComparable for floats. |
class |
GenericWritable
A wrapper for Writable instances. |
class |
IntWritable
A WritableComparable for ints. |
class |
LongWritable
A WritableComparable for longs. |
class |
MD5Hash
A Writable for MD5 hash values. |
class |
NullWritable
Singleton Writable with no data. |
class |
ObjectWritable
A polymorphic Writable that writes an instance with it's class name. |
class |
Text
This class stores text using standard UTF8 encoding. |
class |
TwoDArrayWritable
A Writable for 2D arrays containing a matrix of instances of a class. |
class |
UTF8
Deprecated. replaced by Text |
class |
VersionedWritable
A base class for Writables that provides version checking. |
| Methods in org.apache.hadoop.io that return Writable | |
|---|---|
static Writable |
WritableUtils.clone(Writable orig,
JobConf conf)
Make a copy of a writable object using serialization to a buffer. |
Writable[][] |
TwoDArrayWritable.get()
|
Writable |
GenericWritable.get()
Return the wrapped instance. |
Writable[] |
ArrayWritable.get()
|
Writable |
ArrayFile.Reader.get(long n,
Writable value)
Return the n th value in the file. |
Writable |
MapFile.Reader.get(WritableComparable key,
Writable val)
Return the value for the named key, or null if none exists. |
Writable |
WritableFactory.newInstance()
Return a new instance. |
static Writable |
WritableFactories.newInstance(Class c)
Create a new instance of a class with a defined factory. |
static Writable |
WritableFactories.newInstance(Class c,
Configuration conf)
Create a new instance of a class with a defined factory. |
Writable |
ArrayFile.Reader.next(Writable value)
Read and return the next value in the file. |
| Methods in org.apache.hadoop.io with parameters of type Writable | |
|---|---|
void |
ArrayFile.Writer.append(Writable value)
Append a value to the file. |
void |
MapFile.Writer.append(WritableComparable key,
Writable val)
Append a key/value pair to the map. |
void |
SequenceFile.Writer.append(Writable key,
Writable val)
Append a key/value pair. |
static Writable |
WritableUtils.clone(Writable orig,
JobConf conf)
Make a copy of a writable object using serialization to a buffer. |
Writable |
ArrayFile.Reader.get(long n,
Writable value)
Return the n th value in the file. |
Writable |
MapFile.Reader.get(WritableComparable key,
Writable val)
Return the value for the named key, or null if none exists. |
void |
SequenceFile.Reader.getCurrentValue(Writable val)
Get the 'value' corresponding to the last read 'key'. |
Writable |
ArrayFile.Reader.next(Writable value)
Read and return the next value in the file. |
boolean |
SequenceFile.Reader.next(Writable key)
Read the next key in the file into key , skipping its
value. |
boolean |
MapFile.Reader.next(WritableComparable key,
Writable val)
Read the next key/value pair in the map into key and
val . |
boolean |
SequenceFile.Reader.next(Writable key,
Writable val)
Read the next key/value pair in the file into key and
val . |
void |
GenericWritable.set(Writable obj)
Set the instance that is wrapped. |
void |
ArrayWritable.set(Writable[] values)
|
void |
TwoDArrayWritable.set(Writable[][] values)
|
| Constructors in org.apache.hadoop.io with parameters of type Writable | |
|---|---|
ArrayWritable(Class valueClass,
Writable[] values)
|
|
TwoDArrayWritable(Class valueClass,
Writable[][] values)
|
| Uses of Writable in org.apache.hadoop.ipc |
|---|
| Methods in org.apache.hadoop.ipc that return Writable | |
|---|---|
Writable |
RPC.Server.call(Writable param)
|
abstract Writable |
Server.call(Writable param)
Called for each call. |
Writable[] |
Client.call(Writable[] params,
InetSocketAddress[] addresses)
Makes a set of calls in parallel. |
Writable |
Client.call(Writable param,
InetSocketAddress address)
Make a call, passing param , to the IPC server running at
address , returning the value. |
| Methods in org.apache.hadoop.ipc with parameters of type Writable | |
|---|---|
Writable |
RPC.Server.call(Writable param)
|
abstract Writable |
Server.call(Writable param)
Called for each call. |
Writable[] |
Client.call(Writable[] params,
InetSocketAddress[] addresses)
Makes a set of calls in parallel. |
Writable |
Client.call(Writable param,
InetSocketAddress address)
Make a call, passing param , to the IPC server running at
address , returning the value. |
| Uses of Writable in org.apache.hadoop.mapred |
|---|
| Classes in org.apache.hadoop.mapred that implement Writable | |
|---|---|
class |
ClusterStatus
Summarizes the size and current state of the cluster. |
class |
FileSplit
A section of an input file. |
class |
JobStatus
Describes the current status of a job. |
class |
TaskReport
A report on the state of a task. |
| Methods in org.apache.hadoop.mapred that return Writable | |
|---|---|
Writable |
TextInputFormat.LineRecordReader.createValue()
|
Writable |
SequenceFileRecordReader.createValue()
|
Writable |
RecordReader.createValue()
Create an object of the appropriate type to be used as the value. |
static Writable |
MapFileOutputFormat.getEntry(MapFile.Reader[] readers,
Partitioner partitioner,
WritableComparable key,
Writable value)
Get an entry from output generated by this class. |
| Methods in org.apache.hadoop.mapred with parameters of type Writable | |
|---|---|
boolean |
SequenceFileInputFilter.Filter.accept(Writable key)
filter function Decide if a record should be filtered or not |
boolean |
SequenceFileInputFilter.RegexFilter.accept(Writable key)
Filtering method If key matches the regex, return true; otherwise return false |
boolean |
SequenceFileInputFilter.PercentFilter.accept(Writable key)
Filtering method If record# % frequency==0, return true; otherwise return false |
boolean |
SequenceFileInputFilter.MD5Filter.accept(Writable key)
Filtering method If MD5(key) % frequency==0, return true; otherwise return false |
void |
OutputCollector.collect(WritableComparable key,
Writable value)
Adds a key/value pair to the output. |
protected void |
SequenceFileRecordReader.getCurrentValue(Writable value)
|
static Writable |
MapFileOutputFormat.getEntry(MapFile.Reader[] readers,
Partitioner partitioner,
WritableComparable key,
Writable value)
Get an entry from output generated by this class. |
int |
Partitioner.getPartition(WritableComparable key,
Writable value,
int numPartitions)
Returns the paritition number for a given entry given the total number of partitions. |
void |
Mapper.map(WritableComparable key,
Writable value,
OutputCollector output,
Reporter reporter)
Maps a single input key/value pair into intermediate key/value pairs. |
protected boolean |
SequenceFileRecordReader.next(Writable key)
|
boolean |
TextInputFormat.LineRecordReader.next(Writable key,
Writable value)
Read a line. |
boolean |
SequenceFileRecordReader.next(Writable key,
Writable value)
|
boolean |
RecordReader.next(Writable key,
Writable value)
Reads the next key/value pair. |
void |
TextOutputFormat.LineRecordWriter.write(WritableComparable key,
Writable value)
|
void |
RecordWriter.write(WritableComparable key,
Writable value)
Writes a key/value pair. |
| Uses of Writable in org.apache.hadoop.mapred.lib |
|---|
| Methods in org.apache.hadoop.mapred.lib with parameters of type Writable | |
|---|---|
int |
HashPartitioner.getPartition(WritableComparable key,
Writable value,
int numReduceTasks)
Use Object.hashCode() to partition. |
void |
TokenCountMapper.map(WritableComparable key,
Writable value,
OutputCollector output,
Reporter reporter)
|
void |
RegexMapper.map(WritableComparable key,
Writable value,
OutputCollector output,
Reporter reporter)
|
void |
InverseMapper.map(WritableComparable key,
Writable value,
OutputCollector output,
Reporter reporter)
The inverse function. |
void |
IdentityMapper.map(WritableComparable key,
Writable val,
OutputCollector output,
Reporter reporter)
The identify function. |
| Uses of Writable in org.apache.hadoop.tools |
|---|
| Methods in org.apache.hadoop.tools with parameters of type Writable | |
|---|---|
void |
Logalyzer.LogRegexMapper.map(WritableComparable key,
Writable value,
OutputCollector output,
Reporter reporter)
|
| Uses of Writable in org.apache.hadoop.util |
|---|
| Methods in org.apache.hadoop.util with parameters of type Writable | |
|---|---|
void |
CopyFiles.DFSCopyFilesMapper.map(WritableComparable key,
Writable value,
OutputCollector out,
Reporter reporter)
Map method. |
void |
CopyFiles.HTTPCopyFilesMapper.map(WritableComparable key,
Writable val,
OutputCollector out,
Reporter reporter)
|
|
||||||||||
| PREV NEXT | FRAMES NO FRAMES |


Download Code
Burgers equation
For a given viscosity parameter $\nu$ and for time $t>0$, we consider the 2D Burgers equation on the unit square
with zero Neumann boundary conditions and initial condition
and its numerical approximation using
- Finite Elements in space and Runge-Kutta in time,
- a POD reduction of the FEM model, and
- a DMD reduced order model.
We present the basic ideas and how they are realized in a basic Python script.
0. The Setup
Please download and run the python file burgers.py
to check the
implementation, to reproduce the presented results, and to the behavior of the
routines for different parameters.
Here is the start of the script that lists the modules on which the
implementation bases on.
import dolfin
import scipy.linalg as spla
import matplotlib.pyplot as plt
import numpy as np
from scipy.integrate import solve_ivp
from spacetime_galerkin_pod.ldfnp_ext_cholmod import SparseFactorMassmat
The code uses dolfin
which is the python interface to
FEniCS while the other modules scipy
, numpy
,
and matplotlib
are standard in python, I would say. The module
ldfnp_ext_cholmod
is a little wrapper for the sparsity optimizing Cholesky
decomposition of sksparse.
It is available from my github
repository and falls back
to numpy
routines, in the case that sksparse
is not available.
1. The FEM Discretization
The spatial discretization in FEniCS
First, the mesh is defined – here a uniform triangulation of the unit square
controlled by the parameter N
. For the value N=80
and for globally smooth
and piecewise quadratic ansatz functions, the resulting dimension of the system
is 51842.
# The mesh
pone = dolfin.Point(-1, -1)
ptwo = dolfin.Point(1, 1)
mesh = dolfin.RectangleMesh(pone, ptwo, N, N)
V = dolfin.VectorFunctionSpace(mesh, 'CG', 2)
### The FENICS FEM Discretization
v = dolfin.TestFunction(V)
u = dolfin.TrialFunction(V)
Next, the discrete linear operators are defined and exported as a SciPy sparse
matrix. Also, we factorize the mass matrix mmat
for later use and define the
norm that is weighted with the mass matrix as this is the discrete $L^2$ norm.
# ## the mass matrix
mform = dolfin.inner(v, u)*dolfin.dx
massm = dolfin.assemble(mform)
mmat = dolfin.as_backend_type(massm).sparray()
mmat.eliminate_zeros()
# factorize it for later
mfac = SparseFactorMassmat(mmat)
# norm induced by the mass matrix == discrete L2-norm
def mnorm(uvec):
return np.sqrt(np.inner(uvec, mmat.dot(uvec)))
# ## the stiffness matrix
# as a form in FEniCS
aform = nu*dolfin.inner(dolfin.grad(v), dolfin.grad(u))*dolfin.dx
aassm = dolfin.assemble(aform)
# as a sparse matrix
amat = dolfin.as_backend_type(aassm).sparray()
amat.eliminate_zeros()
Then we define the convective term as function of the velocity u
both in terms
of an FE-function and as a function of the associated coefficient vector.
# ## the convective term
# as a function in FEniCS
def burgers_nonl_func(ufun):
cform = dolfin.inner(dolfin.grad(ufun)*ufun, v)*dolfin.dx
cass = dolfin.assemble(cform)
return cass
# as a vector to form map
def burgers_nonl_vec(uvec):
ufun = dolfin.Function(V)
ufun.vector().set_local(uvec)
bnlform = burgers_nonl_func(ufun)
bnlvec = bnlform.get_local()
return bnlvec
The Runge-Kutta Time Integration
To apply standard time integration schemes (here, RK23
turned out to be most
efficient), we define the right hand side $f_h$ of the spatially discretized problem
In this case the right hand side is an application of the discrete diffusion
and convection operator and the inverse of the mass matrix that, simply
speaking, maps a (discrete) form onto a discrete function. Note that there is no
explicit time dependency in the Burgers equation, but SciPy’s solve_ivp
requires this parameter. Also, we define the initial value here. The time grid
is used to store the solution for the snapshots needed later, but the time
integrator uses his internal time grid.
inivstrg = 'exp(-3.*(x[0]*x[0]+x[1]*x[1]))'
inivexpr = dolfin.Expression((inivstrg, inivstrg), degree=2)
inivfunc = dolfin.interpolate(inivexpr, V)
inivvec = inivfunc.vector().get_local()
burgsol = solve_ivp(brhs, (t0, tE), inivvec, t_eval=timegrid, method='RK23')
fullsol = burgsol.y
Here is the result. Note the sharp front that develops towards the end of the time
integration.
Solution snapshots of the full FEM model
2. POD Reduced Model
If one has snapshots of the solution $v _ h$ at some time instances $t _ i$ one
may well think that the span of the matrix of snapshots
is a good candidate for a space in which the solution evolves in. One may even
go further and look for a low-dimensional basis of this space. The span of a
matrix is best approximated by its dominant singular vectors. And this is the idea of
Proper Orthogonal Decomposition (POD) – use the leading singular vectors as a
basis for the solution space.
We use the Nts=101
snapshots of the FEM solutions to setup the matrix of
measurements $X$ and to compute the POD modes as $M^{-1/2}v _ k$, where $v _ k$
is the $k$-th leading left singular vector of $M^{1/2}X$. This procedure gives a
low-dimensional orthogonal (in the discrete $L^2$ inner product) basis that
optimally parametrizes the subspace of $L^2$ that is spanned by the solution
snapshots. In this example, we use the poddim=25
leading singular vectors
to define the reduced model.
snapshotmat = mfac.Ft.dot(burgsol.y)
podmodes, svals, _ = spla.svd(snapshotmat, full_matrices=False)
selected_podmodes = podmodes[:, :poddim]
podvecs = mfac.solve_Ft(selected_podmodes)
In this implementation we use a sparse factor of the mass matrix instead of the
square root. The singular values (in particular those that correspond to the
discarded directions) give an indication of how good the approximation is.
The singular values of the snapshot matrix
Here, the decay is comparatively slow, so that a one should not expect a good
low dimensional approximation by POD.
For the simulation, the state is parametrized by $u_h (t) \approx V \tilde
u_h(t)$ where $V$ is the matrix of the POD modes (in the code $V$ denoted by
podvecs
), which gives a system in $\tilde u _ h$ with 25 degrees of freedom
(as opposed to the 51842 of the full order model).
redamat = podvecs.T.dot(amat.dot(podvecs)) # the projected stiffness
def redbrhs(time, redvec):
inflatedv = podvecs.dot(redvec)
redconv = podvecs.T.dot(burgers_nonl_vec(inflatedv))
return -redamat.dot(redvec) - redconv.flatten()
Here we define the projected stiffness matrix and the reduced nonlinearity
through
- inflating the reduced state to full dimension
- applying the nonlinearity
- projecting down the result.
This means that our model is not completely independent of the full dimension.
For this problem there are hyperreduction techiques like DEIM.
Thus, the right hand side is readily defined the more that the projected mass
matrix is the identity. Why?
Finally, the initial value is projected into the reduced coordinates and the
reduced system is integrated in time.
redburgsol = solve_ivp(redbrhs, (t0, tE), prjinivvec,
t_eval=timegrid, method='RK23')
podredsol = redburgsol.y
In the solution we see that the reduced order model gives a decent approximation
in the smooth regime in the beginning and has its troubles approximating the front as can be seen in the error (log) plot.
Snapshots of the solution of the reduced system
Snapshots of the log of the error between the full and the POD solution
3. DMD Reduced Model
POD is partially data driven – it uses data to create a basis but still
uses (a projection of) the model. If only snapshots but no model is given, one
may use the method of Dynamic Mode Decomposition (DMD) that tries to identify
a matrix $A$ that evolves the state like
In practice, one uses a set of snapshots and the two measurement matrices
and
Note that $X’$ is basically $X$ shifted by one time step.
Then the DMD matrix can be found by solving the linear regression problem
For the reduced DMD model we use the same snapshot matrix as for the POD. The
regression problem is solved via SVD to compute the needed pseudo inverse since
this naturally allows for a rank reduction and a factored representation of the
DMD matrix.
# ### dmd using truncated svd inverse
fburgsol = burgsol.y
Xmat = fburgsol[:, :-1]
Xdsh = fburgsol[:, 1:]
ux, sx, vxh = spla.svd(Xmat, full_matrices=False)
uxr, sxr, vxhr = ux[:, :poddim], sx[:poddim], vxh[:poddim, :]
# compute the dmd matrix in factored form: `dmda = dmdaone * dmdatwo`
dmdaone = Xdsh.dot(vxhr.T)
dmdatwo = np.linalg.solve(np.diag(sxr), uxr.T)
Once the DMD matrix $A$ is determined, the simulation of the DMD reduced model
is only a repeated multiplication by $A$.
# simulation of the dmd reduced model
dmdxo = inivvec
dmdsol = [dmdxo]
for k in np.arange(Nts):
dmdsol.append(dmdaone.dot(dmdatwo.dot(dmdsol[-1])))
dmdsol = np.array(dmdsol).T
As can be seen from the results and the error plots, DMD does a good job in the
initial phase but fails in the region with the sharp front.
Snapshots of the DMD solution
It is commonly accepted that POD does not work well for transport dominated
problems – like the current case with the low viscosity parameter nu=1e-4
.
So, I think that the results for POD are quite good noting that the reduced
order model has 25 degrees of freedom whereas the full model has 51842.
Nonetheless, in my tests, increasing the number of basis functions did not help
much. One can use a larger nu
to get better POD approximations.
The DMD approach shows a similar performance. If compared to POD, the
qualitative approximation looks less good but the numbers are slightly better.
All in all, the DMD approximation seems less reliable as for some parameter
choices, the performance severely deteriorated.
Differential Equations.* arXiv:1611.04050
Decomposition: Theory and Applications* arXiv:1312.0041


Announcing the 2019 Apache Cassandra Summit
Join the Apache Cassandra community for the 2019 Apache Cassandra Summit, hosted with ApacheCon in Las Vegas, NV (Sept 9 - 13).
Apache Cassandra at ApacheCon
For more information about other events at ApacheCon, see ApacheCon 2019.
Day One: Next Generation Cassandra Conference (NGCC)
NGCC (Next Generation Cassandra Conference), a one-day event for Apache Cassandra contributors and large system operators to meet, discuss, and plan future development in the project. NGCC will take place on Tuesday, September 10th.
NGCC is an advanced event targeted toward Apache Cassandra contributors and large system / platform operators. Content will focus on Cassandra internals and is geared toward those with detailed knowledge of the codebase and architecture. All are welcome to attend.
Day Two: Apache Cassandra Summit
The Apache Cassandra Summit, a one-day event for Apache Cassandra users to meet, network, and learn about what’s new and what’s coming in Cassandra. The Apache Cassandra Summit will be held on Wednesday, Sept 11.
The Apache Cassandra Summit is targeted toward a wider audience. Topics should be interesting and accessible to those whose first introduction to Cassandra is at this event, and those who have been active in the community for many years.
Call for Presentations
We’re excited to announce the Call for Presentations is now open for both, closing Monday, May 13 at 2pm BST (UTC+1).
Apache Cassandra Summit CFP (40-minute presentation, 5-minute Q&A) –
Example proposals might include:
- Lessons learned operating Apache Cassandra at scale.
- Customizations and ways members of the community have extended Apache Cassandra to make it a great fit for their use case.
- Stability improvements, performance enhancements, and new features in an upcoming Cassandra release.
- Something we haven’t thought about that a general audience would be interested to hear.
Next Generation Cassandra Conference CFP (30-minute presentation, 15m breakout) –
Example proposals might include:
- Presentations from contributors and large operators covering pain points and proposals to address them
- Planned or proposed improvements in specific areas of the Apache Cassandra codebase
- Planned or proposed improvements we can make to strengthen and empower the Apache Cassandra community.
- Something we haven’t thought about that advanced operators / contributors would be interested to hear.
Click here to submit a proposal for the Apache Cassandra Summit or NGCC.
When submitting, please ensure you select “Cassandra” as the category. For NGCC submissions, please include “NGCC Proposal” in the “Optional Notes” field.


CURRICLUM VITAE
VIGNESHWAR PASUPATHI
Summary
A energetic and talented Civil Engineer passionate I.T sector
looking for a challenging position as a Full Stack Developer.
Passionate, self-motivated individual with a drive to succeed,
having excellent organizational skills and team player.
Profile
A Civil graduate having a total of 4+years experience as Design Engineer.
Passionate about new technologies and to pursue my dream of becoming a Developer,
Enhanced my skills in I.T field by doing a Full Stack Developer course @ Guvi to enter my dream job.
Looking for a challenging position in the field of design engineering where I can apply my skills and knowledge towards the growth and development of the organisation.


{% extends "layout.html" %} {% block title %}Dashboard{% endblock %} {% block body %}
You do not have access to this resource.


Extracting to jar
file makes it convenient to run a Java program from command
line. But today I encountered a problem of extracting jar
file with IntelliJ Idea.
The problem is that after using the Build Artifact
function in IntelliJ Idea to
build a jar
file, I cannot run it from command line: it says “Error: Could not
find or load main class”.
I searched the Internet and finally found two solutions.
Solution 1: Manually delete some files from the jar
file
Delete from the jar
file these files: META-INF/*.RSA
, META-INF/*.SF
, META-INF/*.DSA
.
This is easy. You can open some archive manager software and delete these files and save. Then the program will run like a magic.
This solution is from Andreas Volkmann. In the solution, he mentioned that this is related to “signatures of interfering dependencies”, which inspired me to have a look at how to avoid these kind of signature of dependencies and leaded to the second solution.
Solution 2: Change extracting library jars to pack as it is
In the configuration of Artifact (Project Structure -> Artifact
), there is a
window for choosing source. What we need to do is to remove all the “Extracted”
things, and double click the library files in “Available Elements” on the right.
This is from Someone on stackoverflow.
Comments
- The problem of “could not find or load main class” here is due to that some dependencies are signed which mess up the META-INF folder.
- There are more advanced ways to excluding signature files.
- It is a good habit to keep library jars as it is when packaging a new jar.