updated core to 7.73

.gitignore

@@ -4,3 +4,5 @@ sites/*/settings*.php
 # Ignore paths that contain user-generated content.
 sites/*/files
 sites/*/private
+
+vendor/*

CHANGELOG.txt

@@ -1,3 +1,702 @@
+Drupal 7.73, 2020-09-16
+-----------------------
+- Fixed security issues:
+   - SA-CORE-2020-007
+
+Drupal 7.72, 2020-06-17
+-----------------------
+- Fixed security issues:
+   - SA-CORE-2020-004
+
+Drupal 7.71, 2020-06-03
+-----------------------
+- Fix for jQuery Form bug in Chromium-based browsers
+- Full support for PHP 7.4
+
+Drupal 7.70, 2020-05-19
+-----------------------
+- Fixed security issues:
+   - SA-CORE-2020-002
+   - SA-CORE-2020-003
+
+Drupal 7.69, 2019-12-18
+-----------------------
+- Fixed security issues:
+   - SA-CORE-2019-012
+
+Drupal 7.68, 2019-12-04
+-----------------------
+- Fixed: Hide toolbar when printing
+- Fixed: Settings returned via ajax are not run through hook_js_alter()
+- Fixed: Use drupal_http_build_query() in drupal_http_request()
+- Fixed: DrupalRequestSanitizer not found fatal error when bootstrap phase order is changed
+- Fixed: Block web.config in .htaccess (and vice-versa)
+- Fixed: Create "scripts" element to align rendering workflow to how "styles" are handled
+- PHP 7.3: Fixed 'Cannot change session id when session is active'
+- PHP 7.1: Fixed 'A non-numeric value encountered in theme_pager()'
+- PHP 7.x: Fixed file.inc generated .htaccess does not cover PHP 7
+- PHP 5.3: Fixed check_plain() 'Invalid multibyte sequence in argument' test failures
+- Fixed: Allow passing data as array to drupal_http_request()
+- Fixed: Skip module_invoke/module_hook in calling hook_watchdog (excessive function_exist)
+- Fixed: HTTP status 200 returned for 'Additional uncaught exception thrown while handling exception'
+- Fixed: theme_table() should take an optional footer variable and produce <tfoot>
+- Fixed: 'uasort() expects parameter 1 to be array, null given in node_view_multiple()'
+- [regression] Fix default.settings.php permission
+
+Drupal 7.67, 2019-05-08
+-----------------------
+- Fixed security issues:
+   - SA-CORE-2019-007
+
+Drupal 7.66, 2019-04-17
+-----------------------
+- Fixed security issues:
+   - SA-CORE-2019-006
+
+Drupal 7.65, 2019-03-20
+-----------------------
+- Fixed security issues:
+   - SA-CORE-2019-004
+
+Drupal 7.64, 2019-02-06
+-----------------------
+- [regression] Unset the 'host' header in drupal_http_request() during redirect
+- Fixed: 7.x does not have Phar protection and Phar tests are failing on Drupal 7
+- Fixed: Notice: Undefined index: display_field in file_field_widget_value() (line 582 of /module/file/file.field.inc)
+- Performance improvement: Registry rebuild should not parse the same file twice in the same request
+- Fixed _registry_update() to clear caches after transaction is committed
+
+Drupal 7.63, 2019-01-16
+-----------------------
+- Fixed a fatal error for some Drush users introduced by SA-CORE-2019-002.
+
+Drupal 7.62, 2019-01-15
+-----------------------
+- Fixed security issues:
+   - SA-CORE-2019-001
+   - SA-CORE-2019-002
+
+Drupal 7.61, 2018-11-07
+-----------------------
+- File upload validation functions and hook_file_validate() implementations are
+  now always passed the correct file URI.
+- The default form cache expiration of 6 hours is now configurable (API
+  addition: https://www.drupal.org/node/2857751).
+- Allowed callers of drupal_http_request() to optionally specify an explicit
+  Host header.
+- Allowed the + character to appear in usernames.
+- PHP 7.2: Fixed Archive_Tar incompatibility.
+- PHP 7.2: Removed deprecated function each().
+- PHP 7.2: Avoid count() calls on uncountable variables.
+- PHP 7.2: Removed deprecated create_function() call.
+- PHP 7.2: Make sure variables are arrays in theme_links().
+- Fixed theme-settings.php not being loaded on cached forms
+- Fixed problem with IE11 & Chrome(PointerEvents enabled) & some Firefox scroll to the top of the page after dragging the bottom item with jquery 1.5 <-> 1.11
+
+Drupal 7.60, 2018-10-18
+------------------------
+- Fixed security issues. See SA-CORE-2018-006.
+
+Drupal 7.59, 2018-04-25
+-----------------------
+- Fixed security issues (remote code execution). See SA-CORE-2018-004.
+
+Drupal 7.58, 2018-03-28
+-----------------------
+- Fixed security issues (remote code execution). See SA-CORE-2018-002.
+
+Drupal 7.57, 2018-02-21
+-----------------------
+- Fixed security issues (multiple vulnerabilities). See SA-CORE-2018-001.
+
+Drupal 7.56, 2017-06-21
+-----------------------
+- Fixed security issues (access bypass). See SA-CORE-2017-003.
+
+Drupal 7.55, 2017-06-07
+-----------------------
+- Fixed incompatibility with PHP versions 7.0.19 and 7.1.5 due to duplicate
+  DATE_RFC7231 definition.
+- Made Drupal core pass all automated tests on PHP 7.1.
+- Allowed services such as Let's Encrypt to work with Drupal on Apache, by
+  making Drupal's .htaccess file allow access to the .well-known directory
+  defined by RFC 5785.
+- Made new Drupal sites work correctly on Apache 2.4 when the mod_access_compat
+  Apache module is disabled.
+- Fixed Drupal's URL-generating functions to always encode '[' and ']' so that
+  the URLs will pass HTML5 validation.
+- Various additional bug fixes.
+- Various API documentation improvements.
+- Additional automated test coverage.
+
+Drupal 7.54, 2017-02-01
+-----------------------
+- Modules are now able to define theme engines (API addition:
+  https://www.drupal.org/node/2826480).
+- Logging of searches can now be disabled (new option in the administrative
+  interface).
+- Added menu tree render structure to (pre-)process hooks for theme_menu_tree()
+  (API addition: https://www.drupal.org/node/2827134).
+- Added new function for determining whether an HTTPS request is being served
+  (API addition: https://www.drupal.org/node/2824590).
+- Fixed incorrect default value for short and medium date formats on the date
+  type configuration page.
+- File validation error message is now removed after subsequent upload of valid
+  file.
+- Numerous bug fixes.
+- Numerous API documentation improvements.
+- Additional performance improvements.
+- Additional automated test coverage.
+
+Drupal 7.53, 2016-12-07
+-----------------------
+- Fixed drag and drop support on newer Chrome/IE 11+ versions after 7.51 update
+  when jQuery is updated to 1.7-1.11.0.
+
+Drupal 7.52, 2016-11-16
+-----------------------
+- Fixed security issues (multiple vulnerabilities). See SA-CORE-2016-005.
+
+Drupal 7.51, 2016-10-05
+-----------------------
+- The Update module now also checks for updates to a disabled theme that is
+  used as an admin theme.
+- Exceptions thrown in dblog_watchdog() are now caught and ignored.
+- Clarified the warning that appears when modules are missing or have moved.
+- Log messages are now XSS filtered on display.
+- Draggable tables now work on touch screen devices.
+- Added a setting for allowing double underscores in CSS identifiers
+  (https://www.drupal.org/node/2810369).
+- If a user navigates away from a page while an Ajax request is running they
+  will no longer get an error message saying "An Ajax HTTP request terminated
+  abnormally".
+- The system_region_list() API function now takes an optional third parameter
+  which allows region name translations to be skipped when they are not needed
+  (API addition: https://www.drupal.org/node/2810365).
+- Numerous performance improvements.
+- Numerous bug fixes.
+- Numerous API documentation improvements.
+- Additional automated test coverage.
+
+Drupal 7.50, 2016-07-07 
+-----------------------
+- Added a new "administer fields" permission for trusted users, which is
+  required in addition to other permissions to use the field UI
+  (https://www.drupal.org/node/2483307).
+- Added clickjacking protection to Drupal core by setting the X-Frame-Options
+  header to SAMEORIGIN by default (https://www.drupal.org/node/2735873).
+- Added support for full UTF-8 (emojis, Asian symbols, mathematical symbols) on
+  MySQL and other database drivers when the site and database are configured to
+  allow it (https://www.drupal.org/node/2761183).
+- Improved performance by avoiding a re-scan of directories when a file is
+  missing; instead, trigger a PHP warning (minor API change:
+  https://www.drupal.org/node/2581445).
+- Made it possible to use any PHP callable in Ajax form callbacks, form API
+  form-building functions, and form API wrapper callbacks (API addition:
+  https://www.drupal.org/node/2761169).
+- Fixed that following a password reset link while logged in leaves users unable
+  to change their password (minor user interface change:
+  https://www.drupal.org/node/2759023).
+- Implemented various fixes for automated test failures on PHP 5.4+ and PHP 7.
+  Drupal core automated tests now pass in these environments.
+- Improved support for PHP 7 by fixing various problems.
+- Fixed various bugs with PHP 5.5+ imagerotate(), including when incorrect
+  color indices are passed in.
+- Fixed a regression introduced in Drupal 7.43 that allowed files uploaded by
+  anonymous users to be lost after form validation errors, and that also caused
+  regressions with certain contributed modules.
+- Fixed a regression introduced in Drupal 7.36 which caused the default value
+  of hidden textarea fields to be ignored.
+- Fixed robots.txt to allow search engines to access CSS, JavaScript and image
+  files.
+- Changed wording on the Update Manager settings page to clarify that the
+  option to check for disabled module updates also applies to uninstalled
+  modules (administrative-facing translatable string change).
+- Changed the help text when editing menu links and configuring URL redirect
+  actions so that it does not reference "Drupal" or the drupal.org website
+  (administrative-facing translatable string change).
+- Fixed the locale safety check that is used to ensure that translations are
+  safe to allow for tokens in the href/src attributes of translated strings.
+- Fixed that URL generation only works on port 80 when using domain based
+  language negotation.
+- Made method="get" forms work inside the administrative overlay. The fix adds
+  a new hidden field to these forms when they appear inside the overlay (minor
+  data structure change).
+- Increased maxlength of menu link title input fields in the node form and
+  menu link form from 128 to 255 characters.
+- Removed meaningless post-check=0 and pre-check=0 cache control headers from
+  Drupal HTTP responses.
+- Added a .editorconfig file to auto-configure editors that support it.
+- Added --directory option to run-tests.sh for easier test discovery of all
+  tests within a project.
+- Made run-tests.sh exit with a failure code when there are test fails or
+  problems running the script.
+- Fixed that cookies from previous tests are still present when a new test
+  starts in DrupalWebTestCase.
+- Improved performance of queries on the {authmap} database table.
+- Fixed handling of missing files and functions inside the registry.
+- Fixed Ajax handling for tableselect form elements that use checkboxes.
+- Fixed a bug which caused ip_address() to return nothing when the client IP
+  address and proxy IP address are the same.
+- Added a new option to format_xml_elements() to allow for already encoded
+  values.
+- Changed the {history} table's node ID field to be an unsigned integer, to
+  match the same field in the {node} table and to prevent errors with very
+  large node IDs.
+- Added an explicit page callback to the "admin/people/create" menu item in the
+  User module (minor data structure change). Previously this automatically
+  inherited the page callback from the parent "admin/people" menu item, which
+  broke contributed modules that override the "admin/people" page.
+- Numerous small bug fixes.
+- Numerous API documentation improvements.
+- Additional automated test coverage.
+
+Drupal 7.44, 2016-06-15
+-----------------------
+- Fixed security issues (privilege escalation). See SA-CORE-2016-002.
+
+Drupal 7.43, 2016-02-24
+-----------------------
+- Fixed security issues (multiple vulnerabilities). See SA-CORE-2016-001.
+
+Drupal 7.42, 2016-02-03
+-----------------------
+- Stopped invoking hook_flush_caches() on every cron run, since some modules
+  use that hook for expensive operations that are only needed on cache clears.
+- Changed the default .htaccess and web.config to block Composer-related files.
+- Added static caching to module_load_include() to improve performance.
+- Fixed double-encoding bugs in select field widgets provided by the Options
+  module. The fix deprecates the 'strip_tags' property on option widgets and
+  replaces it with a new 'strip_tags_and_unescape' property (minor data
+  structure change).
+- Improved MySQL 5.7 support by changing the MySQL database driver to stop
+  using the ANSI SQL mode alias, which has different meanings for different
+  MySQL versions.
+- Fixed a regression introduced in Drupal 7.39 which prevented autocomplete
+  functionality from working on servers that are not configured to
+  automatically recognize index.php.
+- Updated the Archive_Tar PEAR package to the latest 1.4.0 release, to fix bugs
+  with tar file handling on various operating systems.
+- Fixed fatal errors on node preview when a field is displayed in the node
+  teaser but hidden in the full node view. The fix removes a
+  field_attach_prepare_view() call from the node_preview() function since it is
+  redundant with one in the node preview theme layer.
+- Improved the description of the "Trimmed" format option on text fields
+  (translatable string change, and minor UI and data structure change).
+- Numerous small bug fixes.
+- Numerous API documentation improvements.
+- Additional automated test coverage.
+
+Drupal 7.41, 2015-10-21
+-----------------------
+- Fixed security issues (open redirect). See SA-CORE-2015-004.
+
+Drupal 7.40, 2015-10-14
+-----------------------
+- Made Drupal's code for parsing .info files run much faster and use much less
+  memory.
+- Prevented drupal_http_request() from returning an error when it receives a
+  201 through 206 HTTP status code.
+- Added support for autoloading traits via the registry on sites running PHP
+  5.4 or higher.
+- Allowed the user-picture.tpl.php theme template to have HTML classes besides
+  the default "user-picture" class printed in it (markup change).
+- Fixed the URL text filter to convert e-mail addresses with plus signs into
+  mailto: links.
+- Added alternate text to file icons displayed by the File module, to improve
+  accessibility (string change, and minor API addition to theme_file_icon()).
+- Changed one-time login link failure messages to be displayed as errors or
+  warnings as appropriate, rather than as regular status messages (minor UI
+  change and data structure change).
+- Changed the default settings.php configuration to exclude private files from
+  the "404_fast_paths" behavior.
+- Changed the page that displays filter tips for a particular text format, for
+  example filter/tips/full_html, to return "page not found" or "access denied"
+  if the format does not exist or the user does not have access to it. This
+  change adds a new menu item to the Filter module's hook_menu() entry (minor
+  data structure change).
+- Added a new hook, hook_block_cid_parts_alter(), to allow modules to alter the
+  cache keys used for caching a particular block.
+- Made drupal_set_message() display and return messages when "0" is passed in
+  as the message to set.
+- Fixed non-functional "Files displayed by default" setting on file fields.
+- The "worker callback" provided in hook_cron_queue_info() and the "finished"
+  callback specified during batch processing can now be any PHP callable
+  instead of just functions.
+- Prevented drupal_set_time_limit() from decreasing the time limit in the case
+  where the PHP maximum execution time is already unlimited.
+- Changed the default thousand marker for numeric fields from a space ("1 000")
+  to nothing ("1000") (minor UI change: https://www.drupal.org/node/1388376).
+- Prevented malformed theme .info files (without a "name" key) from causing
+  exceptions during menu rebuilds. If an .info file without a "name" key is
+  found in a module or theme directory, Drupal will now use the module or
+  theme's machine name as the display name instead.
+- Made the format column in the {date_format_locale} database table
+  case-sensitive, to match the equivalent column in the {date_formats} table.
+- Fixed a bug in the Statistics module that caused JavaScript files attached to
+  a node while it is being viewed to be omitted from the page.
+- Added an optional 'project:' prefix that can be added to dependencies in a
+  module's .info file to indicate which project the dependency resides in (API
+  addition: https://www.drupal.org/node/2299747).
+- Fixed various bugs that occurred after hooks were invoked early in the Drupal
+  bootstrap and that caused module_implements() and drupal_alter() to cache an
+  incomplete set of hook implementations for later use.
+- Set the X-Content-Type-Options header to "nosniff" when possible, to prevent
+  certain web browsers from picking an unsafe MIME type.
+- Prevented the database API from executing multiple queries at once on MySQL,
+  if the site's PHP version is new enough to do so. This is a secondary defense
+  against SQL injection (API change: https://www.drupal.org/node/2463973).
+- Fixed a bug in the Drupal 6 to Drupal 7 upgrade path which caused the upgrade
+  to fail when there were multiple file records pointing to the same file.
+- Numerous small bug fixes.
+- Numerous API documentation improvements.
+- Additional automated test coverage.
+
+Drupal 7.39, 2015-08-19
+-----------------------
+- Fixed security issues (multiple vulnerabilities). See SA-CORE-2015-003.
+
+Drupal 7.38, 2015-06-17
+-----------------------
+- Fixed security issues (multiple vulnerabilities). See SA-CORE-2015-002.
+
+Drupal 7.37, 2015-05-07
+-----------------------
+- Fixed a regression in Drupal 7.36 which caused certain kinds of content types
+  to become disabled if they were defined by a no-longer-enabled module.
+- Removed a confusing description regarding automatic time zone detection from
+  the user account form (minor UI and data structure change).
+- Allowed custom HTML tags with a dash in the name to pass through filter_xss()
+  when specified in the list of allowed tags.
+- Allowed hook_field_schema() implementations to specify indexes for fields
+  based on a fixed-length column prefix (rather than the entire column), as was
+  already allowed in hook_schema() implementations.
+- Fixed PDO exceptions on PostgreSQL when accessing invalid entity URLs.
+- Added a sites/all/libraries folder to the codebase, with instructions for
+  using it.
+- Added a description to the "Administer text formats and filters" permission
+  on the Permissions page (string change).
+- Numerous small bug fixes.
+- Numerous API documentation improvements.
+- Additional automated test coverage.
+
+Drupal 7.36, 2015-04-01
+-----------------------
+- Added a 'file_public_schema' variable which allows modules that define
+  publicly-accessible streams in hook_stream_wrappers() to bypass file download
+  access checks when processing managed file upload fields.
+- Fixed a bug that caused database query tags not to be added to search-related
+  database queries under many circumstances, and which prevented the
+  corresponding hook_query_TAG_alter() implementations from being called.
+- Fixed the "for" attribute on managed file upload field labels to improve
+  accessibility (minor markup change).
+- Added a 'javascript_always_use_jquery' variable which can be set to FALSE by
+  sites that may not need jQuery loaded on all pages, and a 'requires_jquery'
+  option to drupal_add_js() which modules can set to FALSE when adding
+  JavaScript files that have no dependency on jQuery (API addition:
+  https://www.drupal.org/node/2462717).
+- Fixed incorrect foreign keys in the User module's role_permission and
+  users_roles database tables.
+- Changed permission descriptions throughout Drupal core to consistently link
+  to relevant administrative pages, regardless of whether the user viewing the
+  Permissions page can view the page being linked to (minor UI change).
+- Fixed the drupal_add_region_content() function so that it actually adds
+  content to the page.
+- Added an 'image_suppress_itok_output' variable to allow sites already using
+  the existing 'image_allow_insecure_derivatives' variable to also prevent
+  security tokens from appearing in image derivative URLs.
+- Fixed double-escaping of theme names in the Block module administrative
+  interface (minor string change).
+- Added basic support for Xdebug when running automated tests.
+- Fixed a bug which caused previewing a node to remove elements from the node
+  being edited. With this fix, calling node_preview() will no longer modify the
+  passed-in node object (minor API change).
+- Added a user_has_role() function to check whether a user has a particular
+  role (API addition: https://www.drupal.org/node/2462411).
+- Fixed installation failures when an opcode cache is enabled.
+- Fixed a bug in the Drupal 6 to Drupal 7 upgrade path which caused private
+  files to be inaccessible.
+- Fixed a bug in the Drupal 6 to Drupal 7 upgrade path which caused user
+  pictures to be lost.
+- Fixed missing language code in hook_field_attach_view_alter() when it is
+  invoked from field_view_field().
+- Stopped sending ETag and Last-Modified headers for uncached page requests,
+  since they break caching for certain Varnish and Nginx configurations.
+- Changed the Simpletest module to allow PSR-4 test classes to be used in
+  Drupal 7.
+- Fixed a fatal error that occurred when using the Comment module's "Unpublish
+  comment containing keyword(s)" action.
+- Changed the "lang" attribute on language links to "xml:lang" so it validates
+  as XHTML (minor markup change).
+- Prevented the form API from allowing arrays to be submitted for various form
+  elements, such as textfields, textareas, and password fields (API change:
+  https://www.drupal.org/node/2462723).
+- Fixed a bug in the Contact module which caused the global user object to have
+  the incorrect name and e-mail address during the remainder of the page
+  request after the contact form is submitted.
+- Numerous small bug fixes.
+- Numerous API documentation improvements.
+- Additional automated test coverage.
+
+Drupal 7.35, 2015-03-18
+-----------------------
+- Fixed security issues (multiple vulnerabilities). See SA-CORE-2015-001.
+
+Drupal 7.34, 2014-11-19
+-----------------------
+- Fixed security issues (multiple vulnerabilities). See SA-CORE-2014-006.
+
+Drupal 7.33, 2014-11-07
+-----------------------
+- Began storing the file modification time of each module and theme in the
+  {system} database table so that contributed modules can use it to identify
+  recently changed modules and themes (minor data structure change to the
+  return value of system_get_info() and other related functions).
+- Added a "Did you mean?" feature to the run-tests.sh script for running
+  automated tests from the command line, to help developers who are attempting
+  to run a particular test class or group.
+- Changed the date format used in various HTTP headers output by Drupal core
+  from RFC 1123 format to RFC 7231 format.
+- Added a "block_cache_bypass_node_grants" variable to allow sites which have
+  node access modules enabled to use the block cache if desired (API addition).
+- Made image derivative generation HTTP requests return a 404 error (rather
+  than a 500 error) when the source image does not exist.
+- Fixed a bug which caused user pictures to be removed from the user object
+  after saving, and resulted in data loss if the user account was subsequently
+  re-saved.
+- Fixed a bug in which field_has_data() did not return TRUE for fields that
+  only had data in older entity revisions, leading to loss of the field's data
+  when the field configuration was edited.
+- Fixed a bug which caused the Ajax progress throbber to appear misaligned in
+  many situatons (minor styling change).
+- Prevented the Bartik theme from lower-casing the "Permalink" link on
+  comments, for improved multilingual support (minor UI change).
+- Added a "preferred_menu_links" tag to the database query that is used by
+  menu_link_get_preferred() to find the preferred menu link for a given path,
+  to make it easier to alter.
+- Increased the maximum allowed length of block titles to 255 characters
+  (database schema change to the {block} table).
+- Removed the Field module's field_modules_uninstalled() function, since it did
+  not do anything when it was invoked.
+- Added a "theme_hook_original" variable to templates and theme functions and
+  an optional sitewide theme debug mode, to provide contextual information in
+  the page's HTML to theme developers. The theme debug mode is based on the one
+  used with Twig in Drupal 8 and can be accessed by setting the "theme_debug"
+  variable to TRUE (API addition).
+- Added an entity_view_mode_prepare() API function to allow entity-defining
+  modules to properly invoke hook_entity_view_mode_alter(), and used it
+  throughout Drupal core to fix bugs with the invocation of that hook (API
+  change: https://www.drupal.org/node/2369141).
+- Security improvement: Made the database API's orderBy() method sanitize the
+  sort direction ("ASC" or "DESC") for queries built with db_select(), so that
+  calling code does not have to.
+- Changed the RDF module to consistently output RDF metadata for nodes and
+  comments near where the node is rendered in the HTML (minor markup and data
+  structure change).
+- Added an HTML class to RDFa metatags throughout Drupal to prevent them from
+  accidentally affecting the site appearance (minor markup change).
+- Fixed a bug in the Unicode requirements check which prevented installing
+  Drupal on PHP 5.6.
+- Fixed a bug which caused drupal_get_bootstrap_phase() to abort the bootstrap
+  when called early in the page request.
+- Renamed the "Search result" view mode to "Search result highlighting input"
+  to better reflect how it is used (UI change).
+- Improved database queries generated by EntityFieldQuery in the case where
+  delta or language condition groups are used, to reduce the number of INNER
+  JOINs (this is a minor data structure change affecting code which implements
+  hook_query_alter() on these queries).
+- Removed special-case behavior for file uploads which allowed user #1 to
+  bypass maximum file size and user quota limits.
+- Numerous small bug fixes.
+- Numerous API documentation improvements.
+- Additional automated test coverage.
+
+Drupal 7.32, 2014-10-15
+-----------------------
+- Fixed security issues (SQL injection). See SA-CORE-2014-005.
+
+Drupal 7.31, 2014-08-06
+-----------------------
+- Fixed security issues (denial of service). See SA-CORE-2014-004.
+
+Drupal 7.30, 2014-07-24
+-----------------------
+- Fixed a regression introduced in Drupal 7.29 that caused files or images
+  attached to taxonomy terms to be deleted when the taxonomy term was edited
+  and resaved (and other related bugs with contributed and custom modules).
+- Added a warning on the permissions page to recommend restricting access to
+  the "View site reports" permission to trusted administrators. See
+  DRUPAL-PSA-2014-002.
+- Numerous API documentation improvements.
+- Additional automated test coverage.
+
+Drupal 7.29, 2014-07-16
+-----------------------
+- Fixed security issues (multiple vulnerabilities). See SA-CORE-2014-003.
+
+Drupal 7.28, 2014-05-08
+-----------------------
+- Fixed a regression introduced in Drupal 7.27 that caused JavaScript to break
+  on older browsers (such as Internet Explorer 8 and earlier) when Ajax was
+  used.
+- Increased the timeout used by the Update Manager module when it fetches data
+  from drupal.org (from 5 seconds to 30 seconds), to work around a problem
+  which causes incomplete information about security updates to be presented to
+  site administrators. This fix may lead to a performance slowdown on the
+  Update Manager administration pages, when installing Drupal distributions,
+  and (for sites that use the automated cron feature) on occasional page loads
+  by site visitors.
+- Fixed the behavior of the token system's "[node:summary]" token when the body
+  field does not have a manual summary.
+- Changed the behavior of db_query_temporary() so that it works on SELECT
+  queries even when they have leading comments/whitespace. A side effect of
+  this fix is that db_query_temporary() will now fail with an error if it is
+  ever used on non-SELECT queries.
+- Added a "node_admin_filter" tag to the database query used to build the list
+  of nodes on the content administration page, to make it easier to alter.
+- Made the cron queue system log any exceptions that are thrown while an item
+  in the queue is being processed, rather than stopping the entire PHP request.
+- Improved screen reader support by adding an aria-live HTML attribute to file
+  upload fields when there is an error uploading the file (minor markup
+  change).
+- Made the pager on the Tracker module listing pages show the same number of
+  items as other pagers throughout Drupal core (minor UI change).
+- Fixed a bug which caused caches not to be properly cleared when a file entity
+  was saved or deleted.
+- Added several missing countries to the default list returned by
+  country_get_list() (string change).
+- Replaced the term "weight" with "influence" in the content ranking settings
+  for search, and added help text for administrators (string change).
+- Fixed untranslatable text strings in the administrative interface for the
+  "Crop" effect provided by the Image module (minor string change).
+- Fixed a bug in the Taxonomy module update function introduced in Drupal 7.26
+  that caused memory and CPU problems on sites with very large numbers of
+  unpublished nodes.
+- Numerous small bug fixes.
+- Numerous API documentation improvements.
+- Additional automated test coverage.
+
+Drupal 7.27, 2014-04-16
+-----------------------
+- Fixed security issues (information disclosure). See SA-CORE-2014-002.
+
+Drupal 7.26, 2014-01-15
+-----------------------
+- Fixed security issues (multiple vulnerabilities). See SA-CORE-2014-001.
+
+Drupal 7.25, 2014-01-02
+-----------------------
+- Fixed a bug in node_save() which prevented the saved node from being updated
+  in hook_node_insert() and other similar hooks.
+- Added a meta tag to install.php to prevent it from being indexed by search
+  engines even when Drupal is installed in a subfolder (minor markup change).
+- Fixed a bug in the database API that caused frequent deadlock errors when
+  running merge queries on some servers.
+- Performance improvement: Prevented block rehashing from writing blocks to the
+  database on every cache clear and cron run when the blocks have not changed.
+  This fix results in an extra 'saved' key which is added and set to TRUE for
+  each block returned by _block_rehash() that actually is saved to the database
+  (data structure change).
+- Added an optional 'skip on cron' parameter to hook_cron_queue_info() to allow
+  queues to avoid being automatically processed on cron runs (API addition).
+- Fixed a bug which caused hook_block_view_MODULE_DELTA_alter() to never be
+  invoked if the block delta had a hyphen in it. To implement the hook when the
+  block delta has a hyphen, modules should now replace hyphens with underscores
+  when constructing the function name for the hook implementation.
+- Fixed a bug which caused cached pages to sometimes be sent to the browser
+  with incorrect compression. The fix adds a new 'page_compressed' key to the
+  $cache->data array returned by drupal_page_get_cache() (minor data structure
+  change).
+- Fixed broken tests on PHP 5.5.
+- Made the File and Image modules more robust when saving entities that have
+  deleted files attached. The code in file_field_presave() will now remove the
+  record of the deleted file from the entity before saving (minor data
+  structure change).
+- Standardized menu callback functions throughout Drupal core to return
+  MENU_NOT_FOUND and MENU_ACCESS_DENIED rather than printing their own "page
+  not found" or "access denied" pages (minor API change in the return value of
+  these functions under some circumstances).
+- Fixed a bug in which caches were not properly cleared when a node was deleted
+  via the administrative interface.
+- Changed the Bartik theme to render content contained in <pre>, <code> and
+  similar tags in a larger font size, so it is easier to read.
+- Fixed a bug in the Search module that caused exceptions to be thrown during
+  searches if the server was not configured to represent decimal points as a
+  period.
+- Fixed a regression in the Image module that made image_style_url() not work
+  when a relative path (rather than a complete file URI) was passed to it.
+- Added an optional feature to the Statistics module to allow node views to be
+  tracked by Ajax requests rather than during the server-side generation of the
+  page. This allows the node counter to work on sites that use external page
+  caches (string change and new administrative option:
+  https://drupal.org/node/2164069).
+- Added a link to the drupal.org documentation page for cron to the Cron
+  settings page (string change).
+- Added a 'drupal_anonymous_user_object' variable to allow the anonymous user
+  object returned by drupal_anonymous_user() to be overridden with a classed
+  object (API addition).
+- Changed the database API to allow inserts based on a SELECT * query to work
+  correctly.
+- Changed the database schema of the {file_managed} table to allow Drupal to
+  manage files larger than 4 GB.
+- Changed the File module's hook_field_load() implementation to prevent file
+  entity properties which have the same name as file or image field properties
+  from overwriting the field properties (minor API change).
+- Numerous small bug fixes.
+- Numerous API documentation improvements.
+- Additional automated test coverage.
+
+Drupal 7.24, 2013-11-20
+-----------------------
+- Fixed security issues (multiple vulnerabilities), see SA-CORE-2013-003.
+
+Drupal 7.23, 2013-08-07
+-----------------------
+- Fixed a fatal error on PostgreSQL databases when updating the Taxonomy module
+  from Drupal 6 to Drupal 7.
+- Fixed the default ordering of CSS files for sites using right-to-left
+  languages, to consistently place the right-to-left override file immediately
+  after the CSS it is overriding (API change: https://drupal.org/node/2058463).
+- Added a drupal_check_memory_limit() API function to allow the memory limit to
+  be checked consistently (API addition).
+- Changed the default web.config file for IIS servers to allow favicon.ico
+  files which are present in the filesystem to be accessed.
+- Fixed inconsistent support for the 'tel' protocol in Drupal's URL filtering
+  functions.
+- Performance improvement: Allowed all hooks to be included in the
+  module_implements() cache, even those that are only invoked on HTTP POST
+  requests.
+- Made the database system replace truncate queries with delete queries when
+  inside a transaction, to fix issues with PostgreSQL and other databases.
+- Fixed a bug which caused nested contextual links to display improperly.
+- Fixed a bug which prevented cached image derivatives from being flushed for
+  private files and other non-default file schemes.
+- Fixed drupal_render() to always return an empty string when there is no
+  output, rather than sometimes returning NULL (minor API change).
+- Added protection to cache_clear_all() to ensure that non-cache tables cannot
+  be truncated (API addition: a new isValidBin() method has been added to the
+  default database cache implementation).
+- Changed the default .htaccess file to support HTTP authorization in CGI
+  environments.
+- Changed the password reset form to pre-fill the username when requested via a
+  URL query parameter, and used this in the error message that appears after a
+  failed login attempt (minor data structure and behavior change).
+- Fixed broken support for foreign keys in the field API.
+- Fixed "No active batch" error when a user cancels their own account.
+- Added a description to the "access content overview" permission on the
+  permissions page (string change).
+- Added a drupal_array_diff_assoc_recursive() function to allow associative
+  arrays to be compared recursively (API addition).
+- Added human-readable labels to image styles, in addition to the existing
+  machine-readable name (API change: https://drupal.org/node/2058503).
+- Moved the drupal_get_hash_salt() function to bootstrap.inc and used it in
+  additional places in the code, for added security in the case where there is
+  no hash salt in settings.php.
+- Fixed a regression in Drupal 7.22 that caused internal server errors for
+  sites running on very old Apache 1.x web servers.
+- Numerous small bug fixes.
+- Numerous API documentation improvements.
+- Additional automated test coverage.
 
 Drupal 7.22, 2013-04-03
 -----------------------
@@ -53,7 +752,7 @@ Drupal 7.22, 2013-04-03
   downloaded via the Update Manager (UI change).
 - Added an optional "exclusive" flag to installation profile .info files which
   allows Drupal distributions to force a profile to be selected during
-  installation (API addition).
+  installation (API addition: http://drupal.org/node/1961012).
 - Fixed a bug which caused the database API to not properly close database
   connections.
 - Added a link to the URL for running cron from outside the site to the Cron
@@ -202,8 +901,8 @@ Drupal 7.15, 2012-08-01
 - Numerous API documentation improvements.
 - Additional automated test coverage.
 
-Drupal 7.14 2012-05-02
-----------------------
+Drupal 7.14, 2012-05-02
+-----------------------
 - Fixed "integrity constraint" fatal errors when rebuilding registry.
 - Fixed custom logo and favicon functionality referencing incorrect paths.
 - Fixed DB Case Sensitivity: Allow BINARY attribute in MySQL.
@@ -251,12 +950,12 @@ Drupal 7.14 2012-05-02
   - system_update_7061() converts filepaths too aggressively.
   - Trigger upgrade path: Node triggers removed when upgrading to 7-x from 6.25.
 
-Drupal 7.13 2012-05-02
-----------------------
+Drupal 7.13, 2012-05-02
+-----------------------
 - Fixed security issues (Multiple vulnerabilities), see SA-CORE-2012-002.
 
 Drupal 7.12, 2012-02-01
-----------------------
+-----------------------
 - Fixed bug preventing custom menus from receiving an active trail.
 - Fixed hook_field_delete() no longer invoked during field_purge_data().
 - Fixed bug causing entity info cache to not be cleared with the rest of caches.
@@ -290,11 +989,11 @@ Drupal 7.12, 2012-02-01
   cache.
 
 Drupal 7.11, 2012-02-01
-----------------------
+-----------------------
 - Fixed security issues (Multiple vulnerabilities), see SA-CORE-2012-001.
 
 Drupal 7.10, 2011-12-05
-----------------------
+-----------------------
 - Fixed Content-Language HTTP header to not cause issues with Drush 5.x.
 - Reduce memory usage of theme registry (performance).
 - Fixed PECL upload progress bar for FileField
@@ -647,7 +1346,7 @@ Drupal 7.0, 2011-01-05
       requests.
 
 Drupal 6.23-dev, xxxx-xx-xx (development release)
------------------------
+---------------------------
 
 Drupal 6.22, 2011-05-25
 -----------------------
@@ -657,25 +1356,25 @@ Drupal 6.22, 2011-05-25
 - Fixed a variety of other bugs.
 
 Drupal 6.21, 2011-05-25
-----------------------
+-----------------------
 - Fixed security issues (Cross site scripting), see SA-CORE-2011-001.
 
 Drupal 6.20, 2010-12-15
-----------------------
+-----------------------
 - Fixed a variety of small bugs, improved code documentation.
 
 Drupal 6.19, 2010-08-11
-----------------------
+-----------------------
 - Fixed a variety of small bugs, improved code documentation.
 
 Drupal 6.18, 2010-08-11
-----------------------
+-----------------------
 - Fixed security issues (OpenID authentication bypass, File download access
   bypass, Comment unpublishing bypass, Actions cross site scripting),
   see SA-CORE-2010-002.
 
 Drupal 6.17, 2010-06-02
-----------------------
+-----------------------
 - Improved PostgreSQL compatibility
 - Better PHP 5.3 and PHP 4 compatibility
 - Better browser compatibility of CSS and JS aggregation
@@ -684,7 +1383,7 @@ Drupal 6.17, 2010-06-02
 - Fixed a variety of other bugs.
 
 Drupal 6.16, 2010-03-03
-----------------------
+-----------------------
 - Fixed security issues (Installation cross site scripting, Open redirection,
   Locale module cross site scripting, Blocked user session regeneration),
   see SA-CORE-2010-001.
@@ -696,12 +1395,12 @@ Drupal 6.16, 2010-03-03
 - Fixed a variety of other bugs.
 
 Drupal 6.15, 2009-12-16
-----------------------
+-----------------------
 - Fixed security issues (Cross site scripting), see SA-CORE-2009-009.
 - Fixed a variety of other bugs.
 
 Drupal 6.14, 2009-09-16
-----------------------
+-----------------------
 - Fixed security issues (OpenID association cross site request forgeries,
   OpenID impersonation and File upload), see SA-CORE-2009-008.
 - Changed the system modules page to not run all cache rebuilds; use the
@@ -710,18 +1409,18 @@ Drupal 6.14, 2009-09-16
 - Fixed a variety of small bugs.
 
 Drupal 6.13, 2009-07-01
-----------------------
+-----------------------
 - Fixed security issues (Cross site scripting, Input format access bypass and
   Password leakage in URL), see SA-CORE-2009-007.
 - Fixed a variety of small bugs.
 
 Drupal 6.12, 2009-05-13
-----------------------
+-----------------------
 - Fixed security issues (Cross site scripting), see SA-CORE-2009-006.
 - Fixed a variety of small bugs.
 
 Drupal 6.11, 2009-04-29
-----------------------
+-----------------------
 - Fixed security issues (Cross site scripting and limited information
   disclosure), see SA-CORE-2009-005
 - Fixed performance issues with the menu router cache, the update
@@ -729,7 +1428,7 @@ Drupal 6.11, 2009-04-29
 - Fixed a variety of small bugs.
 
 Drupal 6.10, 2009-02-25
-----------------------
+-----------------------
 - Fixed a security issue, (Local file inclusion on Windows),
   see SA-CORE-2009-003
 - Fixed node_feed() so custom fields can show up in RSS feeds.
@@ -1125,7 +1824,7 @@ Drupal 4.7.9, 2007-12-05
 - fixed a security issue (SQL injection), see SA-2007-031
 
 Drupal 4.7.8, 2007-10-17
-----------------------
+------------------------
 - fixed a security issue (HTTP response splitting), see SA-2007-024
 - fixed a security issue (Cross site scripting via uploads), see SA-2007-026
 - fixed a security issue (API handling of unpublished comment), see SA-2007-030
@@ -1238,7 +1937,7 @@ Drupal 4.6.11, 2007-01-05
 - Fixed security issue (DoS), see SA-2007-002
 
 Drupal 4.6.10, 2006-10-18
-------------------------
+-------------------------
 - Fixed security issue (XSS), see SA-2006-024
 - Fixed security issue (CSRF), see SA-2006-025
 - Fixed security issue (Form action attribute injection), see SA-2006-026

COPYRIGHT.txt

@@ -1,4 +1,4 @@
-All Drupal code is Copyright 2001 - 2012 by the original authors.
+All Drupal code is Copyright 2001 - 2013 by the original authors.
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

INSTALL.mysql.txt

@@ -20,18 +20,21 @@ initial database files. Next you must log in and set the access database rights:
 Again, you will be asked for the 'username' database password. At the MySQL
 prompt, enter the following command:
 
-  GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, INDEX, ALTER
-  ON databasename.*
+  GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, INDEX, ALTER,
+  CREATE TEMPORARY TABLES ON databasename.*
   TO 'username'@'localhost' IDENTIFIED BY 'password';
 
-where
+where:
 
  'databasename' is the name of your database
- 'username@localhost' is the username of your MySQL account
+ 'username' is the username of your MySQL account
+ 'localhost' is the web server host where Drupal is installed
  'password' is the password required for that username
 
-Note: Unless your database user has all of the privileges listed above, you will
-not be able to run Drupal.
+Note: Unless the database user/host combination for your Drupal installation
+has all of the privileges listed above (except possibly CREATE TEMPORARY TABLES,
+which is currently only used by Drupal core automated tests and some
+contributed modules), you will not be able to install or run Drupal.
 
 If successful, MySQL will reply with:
 

INSTALL.txt

@@ -20,8 +20,10 @@ Drupal requires:
   - MySQL 5.0.15 (or greater) (http://www.mysql.com/).
   - MariaDB 5.1.44 (or greater) (http://mariadb.org/). MariaDB is a fully
     compatible drop-in replacement for MySQL.
+  - Percona Server 5.1.70 (or greater) (http://www.percona.com/). Percona
+    Server is a backwards-compatible replacement for MySQL.
   - PostgreSQL 8.3 (or greater) (http://www.postgresql.org/).
-  - SQLite 3.4.2 (or greater) (http://www.sqlite.org/).
+  - SQLite 3.3.7 (or greater) (http://www.sqlite.org/).
 
 For more detailed information about Drupal requirements, including a list of
 PHP extensions and configurations that are required, see "System requirements"

MAINTAINERS.txt

@@ -1,7 +1,8 @@
 
 Drupal core is built and maintained by the Drupal project community. Everyone is
 encouraged to submit issues and changes (patches) to improve Drupal, and to
-contribute in other ways -- see http://drupal.org/contribute to find out how.
+contribute in other ways -- see https://www.drupal.org/contribute to find out
+how.
 
 Branch maintainers
 ------------------
@@ -9,150 +10,152 @@ Branch maintainers
 The Drupal Core branch maintainers oversee the development of Drupal as a whole.
 The branch maintainers for Drupal 7 are:
 
-- Dries Buytaert 'dries' http://drupal.org/user/1
-- Angela Byron 'webchick' http://drupal.org/user/24967
-- David Rothstein 'David_Rothstein' http://drupal.org/user/124982
+- Dries Buytaert 'dries' https://www.drupal.org/u/dries
+- Fabian Franz 'Fabianx' https://www.drupal.org/u/fabianx
+- (provisional) Drew Webber 'mcdruid' https://www.drupal.org/u/mcdruid
 
 
 Component maintainers
 ---------------------
 
 The Drupal Core component maintainers oversee the development of Drupal
-subsystems. See http://drupal.org/contribute/core-maintainers for more
+subsystems. See https://www.drupal.org/contribute/core-maintainers for more
 information on their responsibilities, and to find out how to become a component
 maintainer. Current component maintainers for Drupal 7:
 
 Ajax system
-- Alex Bronstein 'effulgentsia' http://drupal.org/user/78040
-- Earl Miles 'merlinofchaos' http://drupal.org/user/26979
+- Alex Bronstein 'effulgentsia' https://www.drupal.org/u/effulgentsia
+- Earl Miles 'merlinofchaos' https://www.drupal.org/u/merlinofchaos
 
 Base system
-- Károly Négyesi 'chx' http://drupal.org/user/9446
-- Damien Tournoud 'DamZ' http://drupal.org/user/22211
-- Moshe Weitzman 'moshe weitzman' http://drupal.org/user/23
+- Damien Tournoud 'DamZ' https://www.drupal.org/u/damien-tournoud
+- Moshe Weitzman 'moshe weitzman' https://www.drupal.org/u/moshe-weitzman
 
 Batch system
-- Yves Chedemois 'yched' http://drupal.org/user/39567
+- Yves Chedemois 'yched' https://www.drupal.org/u/yched
 
 Cache system
-- Damien Tournoud 'DamZ' http://drupal.org/user/22211
-- Nathaniel Catchpole 'catch' http://drupal.org/user/35733
+- Damien Tournoud 'DamZ' https://www.drupal.org/u/damien-tournoud
+- Nathaniel Catchpole 'catch' https://www.drupal.org/u/catch
 
 Cron system
-- Károly Négyesi 'chx' http://drupal.org/user/9446
-- Derek Wright 'dww' http://drupal.org/user/46549
+- Derek Wright 'dww' https://www.drupal.org/u/dww
 
 Database system
-- Larry Garfield 'Crell' http://drupal.org/user/26398
+- ?
 
   - MySQL driver
-    - Larry Garfield 'Crell' http://drupal.org/user/26398
-    - David Strauss 'David Strauss' http://drupal.org/user/93254
+    - David Strauss 'David Strauss' https://www.drupal.org/u/david-strauss
 
   - PostgreSQL driver
-    - Damien Tournoud 'DamZ' http://drupal.org/user/22211
-    - Josh Waihi 'fiasco' http://drupal.org/user/188162
+    - Damien Tournoud 'DamZ' https://www.drupal.org/u/damien-tournoud
+    - Josh Waihi 'fiasco' https://www.drupal.org/u/josh-waihi
 
   - Sqlite driver
-    - Damien Tournoud 'DamZ' http://drupal.org/user/22211
-    - Károly Négyesi 'chx' http://drupal.org/user/9446
+    - Damien Tournoud 'DamZ' https://www.drupal.org/u/damien-tournoud
 
 Database update system
-- Károly Négyesi 'chx' http://drupal.org/user/9446
-- Ashok Modi 'BTMash' http://drupal.org/user/60422
+- Ashok Modi 'BTMash' https://www.drupal.org/u/btmash
 
 Entity system
-- Wolfgang Ziegler 'fago' http://drupal.org/user/16747
-- Nathaniel Catchpole 'catch' http://drupal.org/user/35733
-- Franz Heinzmann 'Frando' http://drupal.org/user/21850
+- Wolfgang Ziegler 'fago' https://www.drupal.org/u/fago
+- Nathaniel Catchpole 'catch' https://www.drupal.org/u/catch
+- Franz Heinzmann 'Frando' https://www.drupal.org/u/frando
 
 File system
-- Andrew Morton 'drewish' http://drupal.org/user/34869
-- Aaron Winborn 'aaron' http://drupal.org/user/33420
+- Andrew Morton 'drewish' https://www.drupal.org/u/drewish
+- Aaron Winborn 'aaron' https://www.drupal.org/u/aaron
 
 Form system
-- Károly Négyesi 'chx' http://drupal.org/user/9446
-- Alex Bronstein 'effulgentsia' http://drupal.org/user/78040
-- Wolfgang Ziegler 'fago' http://drupal.org/user/16747
-- Daniel F. Kudwien 'sun' http://drupal.org/user/54136
-- Franz Heinzmann 'Frando' http://drupal.org/user/21850
+- Alex Bronstein 'effulgentsia' https://www.drupal.org/u/effulgentsia
+- Wolfgang Ziegler 'fago' https://www.drupal.org/u/fago
+- Daniel F. Kudwien 'sun' https://www.drupal.org/u/sun
+- Franz Heinzmann 'Frando' https://www.drupal.org/u/frando
 
 Image system
-- Andrew Morton 'drewish' http://drupal.org/user/34869
-- Nathan Haug 'quicksketch' http://drupal.org/user/35821
+- Andrew Morton 'drewish' https://www.drupal.org/u/drewish
+- Nathan Haug 'quicksketch' https://www.drupal.org/u/quicksketch
 
 Install system
-- David Rothstein 'David_Rothstein' http://drupal.org/user/124982
+- David Rothstein 'David_Rothstein' https://www.drupal.org/u/david_rothstein
 
 JavaScript
-- Théodore Biadala 'nod_' http://drupal.org/user/598310
-- Steve De Jonghe 'seutje' http://drupal.org/user/264148
-- Jesse Renée Beach 'jessebeach' http://drupal.org/user/748566
+- Théodore Biadala 'nod_' https://www.drupal.org/u/nod_
+- Steve De Jonghe 'seutje' https://www.drupal.org/u/seutje
 
 Language system
-- Francesco Placella 'plach' http://drupal.org/user/183211
-- Daniel F. Kudwien 'sun' http://drupal.org/user/54136
+- Francesco Placella 'plach' https://www.drupal.org/u/plach
+- Daniel F. Kudwien 'sun' https://www.drupal.org/u/sun
 
 Lock system
-- Damien Tournoud 'DamZ' http://drupal.org/user/22211
+- Damien Tournoud 'DamZ' https://www.drupal.org/u/damien-tournoud
 
 Mail system
 - ?
 
 Markup
-- Jacine Luisi 'Jacine' http://drupal.org/user/88931
-- Daniel F. Kudwien 'sun' http://drupal.org/user/54136
+- Jacine Luisi 'Jacine' https://www.drupal.org/u/jacine
+- Daniel F. Kudwien 'sun' https://www.drupal.org/u/sun
 
 Menu system
-- Peter Wolanin 'pwolanin' http://drupal.org/user/49851
-- Károly Négyesi 'chx' http://drupal.org/user/9446
+- Peter Wolanin 'pwolanin' https://www.drupal.org/u/pwolanin
 
 Path system
-- Dave Reid 'davereid' http://drupal.org/user/53892
-- Nathaniel Catchpole 'catch' http://drupal.org/user/35733
+- Dave Reid 'davereid' https://www.drupal.org/u/dave-reid
+- Nathaniel Catchpole 'catch' https://www.drupal.org/u/catch
 
 Render system
-- Moshe Weitzman 'moshe weitzman' http://drupal.org/user/23
-- Alex Bronstein 'effulgentsia' http://drupal.org/user/78040
-- Franz Heinzmann 'Frando' http://drupal.org/user/21850
+- Moshe Weitzman 'moshe weitzman' https://www.drupal.org/u/moshe-weitzman
+- Alex Bronstein 'effulgentsia' https://www.drupal.org/u/effulgentsia
+- Franz Heinzmann 'Frando' https://www.drupal.org/u/frando
 
 Theme system
-- Earl Miles 'merlinofchaos' http://drupal.org/user/26979
-- Alex Bronstein 'effulgentsia' http://drupal.org/user/78040
-- Joon Park 'dvessel' http://drupal.org/user/56782
-- John Albin Wilkins 'JohnAlbin' http://drupal.org/user/32095
+- Earl Miles 'merlinofchaos' https://www.drupal.org/u/merlinofchaos
+- Alex Bronstein 'effulgentsia' https://www.drupal.org/u/effulgentsia
+- Joon Park 'dvessel' https://www.drupal.org/u/dvessel
+- John Albin Wilkins 'JohnAlbin' https://www.drupal.org/u/johnalbin
 
 Token system
-- Dave Reid 'davereid' http://drupal.org/user/53892
+- Dave Reid 'davereid' https://www.drupal.org/u/dave-reid
 
 XML-RPC system
-- Frederic G. Marand 'fgm' http://drupal.org/user/27985
+- Frederic G. Marand 'fgm' https://www.drupal.org/u/fgm
 
 
 Topic coordinators
 ------------------
 
 Accessibility
-- Everett Zufelt 'Everett Zufelt' http://drupal.org/user/406552
-- Brandon Bowersox-Johnson 'bowersox' http://drupal.org/user/186415
+- Everett Zufelt 'Everett Zufelt' https://www.drupal.org/u/everett-zufelt
+- Brandon Bowersox-Johnson 'bowersox' https://www.drupal.org/u/bowersox
 
 Documentation
-- Jennifer Hodgdon 'jhodgdon' http://drupal.org/user/155601
-
-Security
-- Greg Knaddison 'greggles' http://drupal.org/user/36762
+- Jennifer Hodgdon 'jhodgdon' https://www.drupal.org/u/jhodgdon
 
 Translations
-- Gerhard Killesreiter 'killes' http://drupal.org/user/83
+- Gerhard Killesreiter 'killes' https://www.drupal.org/u/gerhard-killesreiter
 
 User experience and usability
-- Roy Scholten 'yoroy' http://drupal.org/user/41502
-- Bojhan Somers 'Bojhan' http://drupal.org/user/87969
+- Roy Scholten 'yoroy' https://www.drupal.org/u/yoroy
+- Bojhan Somers 'Bojhan' https://www.drupal.org/u/bojhan
 
 Node Access
-- Moshe Weitzman 'moshe weitzman' http://drupal.org/user/23
-- Ken Rickard 'agentrickard' http://drupal.org/user/20975
-- Jess Myrbo 'xjm' http://drupal.org/user/65776
+- Moshe Weitzman 'moshe weitzman' https://www.drupal.org/u/moshe-weitzman
+- Ken Rickard 'agentrickard' https://www.drupal.org/u/agentrickard
+
+
+Security team
+-----------------
+
+To report a security issue, see: https://www.drupal.org/security-team/report-issue
+
+The Drupal security team provides Security Advisories for vulnerabilities,
+assists developers in resolving security issues, and provides security
+documentation. See https://www.drupal.org/security-team for more information.
+The security team lead is:
+
+- Michael Hess 'mlhess' https://www.drupal.org/u/mlhess
+
 
 Module maintainers
 ------------------
@@ -161,143 +164,141 @@ Aggregator module
 - ?
 
 Block module
-- John Albin Wilkins 'JohnAlbin' http://drupal.org/user/32095
+- John Albin Wilkins 'JohnAlbin' https://www.drupal.org/u/johnalbin
 
 Blog module
 - ?
 
 Book module
-- Peter Wolanin 'pwolanin' http://drupal.org/user/49851
+- Peter Wolanin 'pwolanin' https://www.drupal.org/u/pwolanin
 
 Color module
 - ?
 
 Comment module
-- Nathaniel Catchpole 'catch' http://drupal.org/user/35733
+- Nathaniel Catchpole 'catch' https://www.drupal.org/u/catch
 
 Contact module
-- Dave Reid 'davereid' http://drupal.org/user/53892
+- Dave Reid 'davereid' https://www.drupal.org/u/dave-reid
 
 Contextual module
-- Daniel F. Kudwien 'sun' http://drupal.org/user/54136
+- Daniel F. Kudwien 'sun' https://www.drupal.org/u/sun
 
 Dashboard module
 - ?
 
 Database logging module
-- Khalid Baheyeldin 'kbahey' http://drupal.org/user/4063
+- Khalid Baheyeldin 'kbahey' https://www.drupal.org/u/kbahey
 
 Field module
-- Yves Chedemois 'yched' http://drupal.org/user/39567
-- Barry Jaspan 'bjaspan' http://drupal.org/user/46413
+- Yves Chedemois 'yched' https://www.drupal.org/u/yched
+- Barry Jaspan 'bjaspan' https://www.drupal.org/u/bjaspan
 
 Field UI module
-- Yves Chedemois 'yched' http://drupal.org/user/39567
+- Yves Chedemois 'yched' https://www.drupal.org/u/yched
 
 File module
-- Aaron Winborn 'aaron' http://drupal.org/user/33420
+- Aaron Winborn 'aaron' https://www.drupal.org/u/aaron
 
 Filter module
-- Daniel F. Kudwien 'sun' http://drupal.org/user/54136
+- Daniel F. Kudwien 'sun' https://www.drupal.org/u/sun
 
 Forum module
-- Lee Rowlands 'larowlan' http://drupal.org/user/395439
+- Lee Rowlands 'larowlan' https://www.drupal.org/u/larowlan
 
 Help module
 - ?
 
 Image module
-- Nathan Haug 'quicksketch' http://drupal.org/user/35821
+- Nathan Haug 'quicksketch' https://www.drupal.org/u/quicksketch
 
 Locale module
-- Gábor Hojtsy 'Gábor Hojtsy' http://drupal.org/user/4166
+- Gábor Hojtsy 'Gábor Hojtsy' https://www.drupal.org/u/gábor-hojtsy
 
 Menu module
 - ?
 
 Node module
-- Moshe Weitzman 'moshe weitzman' http://drupal.org/user/23
-- David Strauss 'David Strauss' http://drupal.org/user/93254
+- Moshe Weitzman 'moshe weitzman' https://www.drupal.org/u/moshe-weitzman
+- David Strauss 'David Strauss' https://www.drupal.org/u/david-strauss
 
 OpenID module
-- Vojtech Kusy 'wojtha' http://drupal.org/user/56154
-- Christian Schmidt 'c960657' http://drupal.org/user/216078
-- Damien Tournoud 'DamZ' http://drupal.org/user/22211
+- Vojtech Kusy 'wojtha' https://www.drupal.org/u/wojtha
+- Christian Schmidt 'c960657' https://www.drupal.org/u/c960657
+- Damien Tournoud 'DamZ' https://www.drupal.org/u/damien-tournoud
 
 Overlay module
-- Katherine Senzee 'ksenzee' http://drupal.org/user/139855
+- Katherine Senzee 'ksenzee' https://www.drupal.org/u/ksenzee
 
 Path module
-- Dave Reid 'davereid' http://drupal.org/user/53892
+- Dave Reid 'davereid' https://www.drupal.org/u/dave-reid
 
 PHP module
 - ?
 
 Poll module
-- Andrei Mateescu 'amateescu' http://drupal.org/user/729614
+- Andrei Mateescu 'amateescu' https://www.drupal.org/u/amateescu
 
 Profile module
 - ?
 
 RDF module
-- Stéphane Corlosquet 'scor' http://drupal.org/user/52142
+- Stéphane Corlosquet 'scor' https://www.drupal.org/u/scor
 
 Search module
-- Doug Green 'douggreen' http://drupal.org/user/29191
+- Doug Green 'douggreen' https://www.drupal.org/u/douggreen
 
 Shortcut module
-- David Rothstein 'David_Rothstein' http://drupal.org/user/124982
+- David Rothstein 'David_Rothstein' https://www.drupal.org/u/david_rothstein
 
 Simpletest module
-- Jimmy Berry 'boombatower' http://drupal.org/user/214218
-- Károly Négyesi 'chx' http://drupal.org/user/9446
+- Jimmy Berry 'boombatower' https://www.drupal.org/u/boombatower
 
 Statistics module
-- Tim Millwood 'timmillwood' http://drupal.org/user/227849
+- Tim Millwood 'timmillwood' https://www.drupal.org/u/timmillwood
 
 Syslog module
-- Khalid Baheyeldin 'kbahey' http://drupal.org/user/4063
+- Khalid Baheyeldin 'kbahey' https://www.drupal.org/u/kbahey
 
 System module
 - ?
 
 Taxonomy module
-- Jess Myrbo 'xjm' http://drupal.org/user/65776
-- Nathaniel Catchpole 'catch' http://drupal.org/user/35733
-- Benjamin Doherty 'bangpound' http://drupal.org/user/100456
+- Nathaniel Catchpole 'catch' https://www.drupal.org/u/catch
+- Benjamin Doherty 'bangpound' https://www.drupal.org/u/bangpound
 
 Toolbar module
 - ?
 
 Tracker module
-- David Strauss 'David Strauss' http://drupal.org/user/93254
+- David Strauss 'David Strauss' https://www.drupal.org/u/david-strauss
 
 Translation module
-- Francesco Placella 'plach' http://drupal.org/user/183211
+- Francesco Placella 'plach' https://www.drupal.org/u/plach
 
 Trigger module
 - ?
 
 Update module
-- Derek Wright 'dww' http://drupal.org/user/46549
+- Derek Wright 'dww' https://www.drupal.org/u/dww
 
 User module
-- Moshe Weitzman 'moshe weitzman' http://drupal.org/user/23
-- David Strauss 'David Strauss' http://drupal.org/user/93254
+- Moshe Weitzman 'moshe weitzman' https://www.drupal.org/u/moshe-weitzman
+- David Strauss 'David Strauss' https://www.drupal.org/u/david-strauss
 
 
 Theme maintainers
 -----------------
 
 Bartik theme
-- Jen Simmons 'jensimmons' http://drupal.org/user/140882
-- Jeff Burns 'Jeff Burnz' http://drupal.org/user/61393
+- Jen Simmons 'jensimmons' https://www.drupal.org/u/jensimmons
+- Jeff Burns 'Jeff Burnz' https://www.drupal.org/u/jeff-burnz
 
 Garland theme
-- John Albin Wilkins 'JohnAlbin' http://drupal.org/user/32095
+- John Albin Wilkins 'JohnAlbin' https://www.drupal.org/u/johnalbin
 
 Seven theme
-- Jeff Burns 'Jeff Burnz' http://drupal.org/user/61393
+- Jeff Burns 'Jeff Burnz' https://www.drupal.org/u/jeff-burnz
 
 Stark theme
-- John Albin Wilkins 'JohnAlbin' http://drupal.org/user/32095
+- John Albin Wilkins 'JohnAlbin' https://www.drupal.org/u/johnalbin

README.txt

@@ -71,12 +71,12 @@ profiles/your_site_profile/themes respectively to restrict their usage to only
 sites that were installed with that specific profile.
 
 More about installation profiles and distributions:
-* Read about the difference between installation profiles and distributions:
-  http://drupal.org/node/1089736
-* Download contributed installation profiles and distributions:
-  http://drupal.org/project/distributions
-* Develop your own installation profile or distribution:
-  http://drupal.org/developing/distributions
+ * Read about the difference between installation profiles and distributions:
+   http://drupal.org/node/1089736
+ * Download contributed installation profiles and distributions:
+   http://drupal.org/project/distributions
+ * Develop your own installation profile or distribution:
+   http://drupal.org/developing/distributions
 
 APPEARANCE
 ----------

UPGRADE.txt

@@ -64,6 +64,9 @@ following the instructions in the INTRODUCTION section at the top of this file:
    Sometimes an update includes changes to default.settings.php (this will be
    noted in the release notes). If that's the case, follow these steps:
 
+   - Locate your settings.php file in the /sites/* directory. (Typically
+     sites/default.)
+
    - Make a backup copy of your settings.php file, with a different file name.
 
    - Make a copy of the new default.settings.php file, and name the copy
@@ -74,6 +77,13 @@ following the instructions in the INTRODUCTION section at the top of this file:
      database information, and you will also want to copy in any other
      customizations you have added.
 
+   You can find the release notes for your version at
+   https://www.drupal.org/project/drupal. At bottom of the project page under
+   "Downloads" use the link for your version of Drupal to view the release
+   notes. If your version is not listed, use the 'View all releases' link. From
+   this page you can scroll down or use the filter to find your version and its
+   release notes.
+
 4. Download the latest Drupal 7.x release from http://drupal.org to a
    directory outside of your web root. Extract the archive and copy the files
    into your Drupal directory.

includes/ajax.inc

@@ -211,7 +211,7 @@
  *
  * When returning an Ajax command array, it is often useful to have
  * status messages rendered along with other tasks in the command array.
- * In that case the the Ajax commands array may be constructed like this:
+ * In that case the Ajax commands array may be constructed like this:
  * @code
  *   $commands = array();
  *   $commands[] = ajax_command_replace(NULL, $output);
@@ -230,6 +230,10 @@
  *   functions.
  */
 function ajax_render($commands = array()) {
+  // Although ajax_deliver() does this, some contributed and custom modules
+  // render Ajax responses without using that delivery callback.
+  ajax_set_verification_header();
+
   // Ajax responses aren't rendered with html.tpl.php, so we have to call
   // drupal_get_css() and drupal_get_js() here, in order to have new files added
   // during this request to be loaded by the page. We only want to send back
@@ -276,7 +280,7 @@ function ajax_render($commands = array()) {
 
   $extra_commands = array();
   if (!empty($styles)) {
-    $extra_commands[] = ajax_command_prepend('head', $styles);
+    $extra_commands[] = ajax_command_add_css($styles);
   }
   if (!empty($scripts_header)) {
     $extra_commands[] = ajax_command_prepend('head', $scripts_header);
@@ -290,9 +294,10 @@ function ajax_render($commands = array()) {
 
   // Now add a command to merge changes and additions to Drupal.settings.
   $scripts = drupal_add_js();
+  drupal_alter('js', $scripts);
   if (!empty($scripts['settings'])) {
     $settings = $scripts['settings'];
-    array_unshift($commands, ajax_command_settings(call_user_func_array('array_merge_recursive', $settings['data']), TRUE));
+    array_unshift($commands, ajax_command_settings(drupal_array_merge_deep_array($settings['data']), TRUE));
   }
 
   // Allow modules to alter any Ajax response.
@@ -308,10 +313,11 @@ function ajax_render($commands = array()) {
  * pulls the form info from $_POST.
  *
  * @return
- *   An array containing the $form and $form_state. Use the list() function
- *   to break these apart:
+ *   An array containing the $form, $form_state, $form_id, $form_build_id and an
+ *   initial list of Ajax $commands. Use the list() function to break these
+ *   apart:
  *   @code
- *     list($form, $form_state, $form_id, $form_build_id) = ajax_get_form();
+ *     list($form, $form_state, $form_id, $form_build_id, $commands) = ajax_get_form();
  *   @endcode
  */
 function ajax_get_form() {
@@ -331,6 +337,17 @@ function ajax_get_form() {
     drupal_exit();
   }
 
+  // When a page level cache is enabled, the form-build id might have been
+  // replaced from within form_get_cache. If this is the case, it is also
+  // necessary to update it in the browser by issuing an appropriate Ajax
+  // command.
+  $commands = array();
+  if (isset($form['#build_id_old']) && $form['#build_id_old'] != $form['#build_id']) {
+    // If the form build ID has changed, issue an Ajax command to update it.
+    $commands[] = ajax_command_update_build_id($form);
+    $form_build_id = $form['#build_id'];
+  }
+
   // Since some of the submit handlers are run, redirects need to be disabled.
   $form_state['no_redirect'] = TRUE;
 
@@ -345,7 +362,7 @@ function ajax_get_form() {
   $form_state['input'] = $_POST;
   $form_id = $form['#form_id'];
 
-  return array($form, $form_state, $form_id, $form_build_id);
+  return array($form, $form_state, $form_id, $form_build_id, $commands);
 }
 
 /**
@@ -366,7 +383,7 @@ function ajax_get_form() {
  * @see system_menu()
  */
 function ajax_form_callback() {
-  list($form, $form_state) = ajax_get_form();
+  list($form, $form_state, $form_id, $form_build_id, $commands) = ajax_get_form();
   drupal_process_form($form['#form_id'], $form, $form_state);
 
   // We need to return the part of the form (or some other content) that needs
@@ -378,8 +395,20 @@ function ajax_form_callback() {
   if (!empty($form_state['triggering_element'])) {
     $callback = $form_state['triggering_element']['#ajax']['callback'];
   }
-  if (!empty($callback) && function_exists($callback)) {
-    return $callback($form, $form_state);
+  if (!empty($callback) && is_callable($callback)) {
+    $result = $callback($form, $form_state);
+
+    if (!(is_array($result) && isset($result['#type']) && $result['#type'] == 'ajax')) {
+      // Turn the response into a #type=ajax array if it isn't one already.
+      $result = array(
+        '#type' => 'ajax',
+        '#commands' => ajax_prepare_response($result),
+      );
+    }
+
+    $result['#commands'] = array_merge($commands, $result['#commands']);
+
+    return $result;
   }
 }
 
@@ -463,6 +492,9 @@ function ajax_deliver($page_callback_result) {
     }
   }
 
+  // Let ajax.js know that this response is safe to process.
+  ajax_set_verification_header();
+
   // Print the response.
   $commands = ajax_prepare_response($page_callback_result);
   $json = ajax_render($commands);
@@ -552,6 +584,29 @@ function ajax_prepare_response($page_callback_result) {
   return $commands;
 }
 
+/**
+ * Sets a response header for ajax.js to trust the response body.
+ *
+ * It is not safe to invoke Ajax commands within user-uploaded files, so this
+ * header protects against those being invoked.
+ *
+ * @see Drupal.ajax.options.success()
+ */
+function ajax_set_verification_header() {
+  $added = &drupal_static(__FUNCTION__);
+
+  // User-uploaded files cannot set any response headers, so a custom header is
+  // used to indicate to ajax.js that this response is safe. Note that most
+  // Ajax requests bound using the Form API will be protected by having the URL
+  // flagged as trusted in Drupal.settings, so this header is used only for
+  // things like custom markup that gets Ajax behaviors attached.
+  if (empty($added)) {
+    drupal_add_http_header('X-Drupal-Ajax-Token', '1');
+    // Avoid sending the header twice.
+    $added = TRUE;
+  }
+}
+
 /**
  * Performs end-of-Ajax-request tasks.
  *
@@ -740,7 +795,12 @@ function ajax_pre_render_element($element) {
 
     $element['#attached']['js'][] = array(
       'type' => 'setting',
-      'data' => array('ajax' => array($element['#id'] => $settings)),
+      'data' => array(
+        'ajax' => array($element['#id'] => $settings),
+        'urlIsAjaxTrusted' => array(
+          $settings['url'] => TRUE,
+        ),
+      ),
     );
 
     // Indicate that Ajax processing was successful.
@@ -1210,3 +1270,49 @@ function ajax_command_restripe($selector) {
     'selector' => $selector,
   );
 }
+
+/**
+ * Creates a Drupal Ajax 'update_build_id' command.
+ *
+ * This command updates the value of a hidden form_build_id input element on a
+ * form. It requires the form passed in to have keys for both the old build ID
+ * in #build_id_old and the new build ID in #build_id.
+ *
+ * The primary use case for this Ajax command is to serve a new build ID to a
+ * form served from the cache to an anonymous user, preventing one anonymous
+ * user from accessing the form state of another anonymous users on Ajax enabled
+ * forms.
+ *
+ * @param $form
+ *   The form array representing the form whose build ID should be updated.
+ */
+function ajax_command_update_build_id($form) {
+  return array(
+    'command' => 'updateBuildId',
+    'old' => $form['#build_id_old'],
+    'new' => $form['#build_id'],
+  );
+}
+
+/**
+ * Creates a Drupal Ajax 'add_css' command.
+ *
+ * This method will add css via ajax in a cross-browser compatible way.
+ *
+ * This command is implemented by Drupal.ajax.prototype.commands.add_css()
+ * defined in misc/ajax.js.
+ *
+ * @param $styles
+ *   A string that contains the styles to be added.
+ *
+ * @return
+ *   An array suitable for use with the ajax_render() function.
+ *
+ * @see misc/ajax.js
+ */
+function ajax_command_add_css($styles) {
+  return array(
+    'command' => 'add_css',
+    'data' => $styles,
+  );
+}

includes/batch.inc

@@ -460,10 +460,10 @@ function _batch_finished() {
       if (isset($batch_set['file']) && is_file($batch_set['file'])) {
         include_once DRUPAL_ROOT . '/' . $batch_set['file'];
       }
-      if (function_exists($batch_set['finished'])) {
+      if (is_callable($batch_set['finished'])) {
         $queue = _batch_queue($batch_set);
         $operations = $queue->getAllItems();
-        $batch_set['finished']($batch_set['success'], $batch_set['results'], $operations, format_interval($batch_set['elapsed'] / 1000));
+        call_user_func($batch_set['finished'], $batch_set['success'], $batch_set['results'], $operations, format_interval($batch_set['elapsed'] / 1000));
       }
     }
   }
@@ -478,18 +478,17 @@ function _batch_finished() {
         $queue->deleteQueue();
       }
     }
+    // Clean-up the session. Not needed for CLI updates.
+    if (isset($_SESSION)) {
+      unset($_SESSION['batches'][$batch['id']]);
+      if (empty($_SESSION['batches'])) {
+        unset($_SESSION['batches']);
+      }
+    }
   }
   $_batch = $batch;
   $batch = NULL;
 
-  // Clean-up the session. Not needed for CLI updates.
-  if (isset($_SESSION)) {
-    unset($_SESSION['batches'][$batch['id']]);
-    if (empty($_SESSION['batches'])) {
-      unset($_SESSION['batches']);
-    }
-  }
-
   // Redirect if needed.
   if ($_batch['progressive']) {
     // Revert the 'destination' that was saved in batch_process().

includes/cache.inc

@@ -14,6 +14,7 @@
  *
  * @param $bin
  *   The cache bin for which the cache object should be returned.
+ *
  * @return DrupalCacheInterface
  *   The cache object associated with the specified bin.
  *
@@ -98,9 +99,11 @@ function cache_get_multiple(array &$cids, $bin = 'cache') {
  * @param $data
  *   The data to store in the cache. Complex data types will be automatically
  *   serialized before insertion. Strings will be stored as plain text and are
- *   not serialized.
+ *   not serialized. Some storage engines only allow objects up to a maximum of
+ *   1MB in size to be stored by default. When caching large arrays or similar,
+ *   take care to ensure $data does not exceed this size.
  * @param $bin
- *   The cache bin to store the data in. Valid core values are:
+ *   (optional) The cache bin to store the data in. Valid core values are:
  *   - cache: (default) Generic cache storage bin (used for theme registry,
  *     locale date, list of simpletest tests, etc.).
  *   - cache_block: Stores the content of various blocks.
@@ -119,7 +122,12 @@ function cache_get_multiple(array &$cids, $bin = 'cache') {
  *     the administrator panel.
  *   - cache_path: Stores the system paths that have an alias.
  * @param $expire
- *   One of the following values:
+ *   (optional) Controls the maximum lifetime of this cache entry. Note that
+ *   caches might be subject to clearing at any time, so this setting does not
+ *   guarantee a minimum lifetime. With this in mind, the cache should not be
+ *   used for data that must be kept during a cache clear, like sessions.
+ *
+ *   Use one of the following values:
  *   - CACHE_PERMANENT: Indicates that the item should never be removed unless
  *     explicitly told to using cache_clear_all() with a cache ID.
  *   - CACHE_TEMPORARY: Indicates that the item should be removed at the next
@@ -137,18 +145,20 @@ function cache_set($cid, $data, $bin = 'cache', $expire = CACHE_PERMANENT) {
 /**
  * Expires data from the cache.
  *
- * If called without arguments, expirable entries will be cleared from the
- * cache_page and cache_block bins.
+ * If called with the arguments $cid and $bin set to NULL or omitted, then
+ * expirable entries will be cleared from the cache_page and cache_block bins,
+ * and the $wildcard argument is ignored.
  *
  * @param $cid
- *   If set, the cache ID to delete. Otherwise, all cache entries that can
- *   expire are deleted.
+ *   If set, the cache ID or an array of cache IDs. Otherwise, all cache entries
+ *   that can expire are deleted. The $wildcard argument will be ignored if set
+ *   to NULL.
  * @param $bin
  *   If set, the cache bin to delete from. Mandatory argument if $cid is set.
  * @param $wildcard
- *   If TRUE, cache IDs starting with $cid are deleted in addition to the
- *   exact cache ID specified by $cid. If $wildcard is TRUE and $cid is '*',
- *   the entire cache bin is emptied.
+ *   If TRUE, the $cid argument must contain a string value and cache IDs
+ *   starting with $cid are deleted in addition to the exact cache ID specified
+ *   by $cid. If $wildcard is TRUE and $cid is '*', the entire cache is emptied.
  */
 function cache_clear_all($cid = NULL, $bin = NULL, $wildcard = FALSE) {
   if (!isset($cid) && !isset($bin)) {
@@ -217,13 +227,6 @@ function cache_is_empty($bin) {
  * @see DrupalDatabaseCache
  */
 interface DrupalCacheInterface {
-  /**
-   * Constructs a new cache interface.
-   *
-   * @param $bin
-   *   The cache bin for which the object is created.
-   */
-  function __construct($bin);
 
   /**
    * Returns data from the persistent cache.
@@ -259,10 +262,17 @@ interface DrupalCacheInterface {
    *   The cache ID of the data to store.
    * @param $data
    *   The data to store in the cache. Complex data types will be automatically
-   *   serialized before insertion.
-   *   Strings will be stored as plain text and not serialized.
+   *   serialized before insertion. Strings will be stored as plain text and not
+   *   serialized. Some storage engines only allow objects up to a maximum of
+   *   1MB in size to be stored by default. When caching large arrays or
+   *   similar, take care to ensure $data does not exceed this size.
    * @param $expire
-   *   One of the following values:
+   *   (optional) Controls the maximum lifetime of this cache entry. Note that
+   *   caches might be subject to clearing at any time, so this setting does not
+   *   guarantee a minimum lifetime. With this in mind, the cache should not be
+   *   used for data that must be kept during a cache clear, like sessions.
+   *
+   *   Use one of the following values:
    *   - CACHE_PERMANENT: Indicates that the item should never be removed unless
    *     explicitly told to using cache_clear_all() with a cache ID.
    *   - CACHE_TEMPORARY: Indicates that the item should be removed at the next
@@ -280,12 +290,14 @@ interface DrupalCacheInterface {
    * cache_page and cache_block bins.
    *
    * @param $cid
-   *   If set, the cache ID to delete. Otherwise, all cache entries that can
-   *   expire are deleted.
+   *   If set, the cache ID or an array of cache IDs. Otherwise, all cache
+   *   entries that can expire are deleted. The $wildcard argument will be
+   *   ignored if set to NULL.
    * @param $wildcard
-   *   If set to TRUE, the $cid is treated as a substring
-   *   to match rather than a complete ID. The match is a right hand
-   *   match. If '*' is given as $cid, the bin $bin will be emptied.
+   *   If TRUE, the $cid argument must contain a string value and cache IDs
+   *   starting with $cid are deleted in addition to the exact cache ID
+   *   specified by $cid. If $wildcard is TRUE and $cid is '*', the entire
+   *   cache is emptied.
    */
   function clear($cid = NULL, $wildcard = FALSE);
 
@@ -311,7 +323,10 @@ class DrupalDatabaseCache implements DrupalCacheInterface {
   protected $bin;
 
   /**
-   * Constructs a new DrupalDatabaseCache object.
+   * Constructs a DrupalDatabaseCache object.
+   *
+   * @param $bin
+   *   The cache bin for which the object is created.
    */
   function __construct($bin) {
     $this->bin = $bin;
@@ -505,7 +520,16 @@ class DrupalDatabaseCache implements DrupalCacheInterface {
     else {
       if ($wildcard) {
         if ($cid == '*') {
-          db_truncate($this->bin)->execute();
+          // Check if $this->bin is a cache table before truncating. Other
+          // cache_clear_all() operations throw a PDO error in this situation,
+          // so we don't need to verify them first. This ensures that non-cache
+          // tables cannot be truncated accidentally.
+          if ($this->isValidBin()) {
+            db_truncate($this->bin)->execute();
+          }
+          else {
+            throw new Exception(t('Invalid or missing cache bin specified: %bin', array('%bin' => $this->bin)));
+          }
         }
         else {
           db_delete($this->bin)
@@ -542,4 +566,25 @@ class DrupalDatabaseCache implements DrupalCacheInterface {
       ->fetchField();
     return empty($result);
   }
+
+  /**
+   * Checks if $this->bin represents a valid cache table.
+   *
+   * This check is required to ensure that non-cache tables are not truncated
+   * accidentally when calling cache_clear_all().
+   *
+   * @return boolean
+   */
+  function isValidBin() {
+    if ($this->bin == 'cache' || substr($this->bin, 0, 6) == 'cache_') {
+      // Skip schema check for bins with standard table names.
+      return TRUE;
+    }
+    // These fields are required for any cache table.
+    $fields = array('cid', 'data', 'expire', 'created', 'serialized');
+    // Load the table schema.
+    $schema = drupal_get_schema($this->bin);
+    // Confirm that all fields are present.
+    return isset($schema['fields']) && !array_diff($fields, array_keys($schema['fields']));
+  }
 }

includes/database/database.inc

@@ -28,18 +28,21 @@
  * Most Drupal database SELECT queries are performed by a call to db_query() or
  * db_query_range(). Module authors should also consider using the PagerDefault
  * Extender for queries that return results that need to be presented on
- * multiple pages, and the Tablesort Extender for generating appropriate queries
- * for sortable tables.
+ * multiple pages (see https://drupal.org/node/508796), and the TableSort
+ * Extender for generating appropriate queries for sortable tables
+ * (see https://drupal.org/node/1848372).
  *
  * For example, one might wish to return a list of the most recent 10 nodes
  * authored by a given user. Instead of directly issuing the SQL query
  * @code
- * SELECT n.nid, n.title, n.created FROM node n WHERE n.uid = $uid LIMIT 0, 10;
+ * SELECT n.nid, n.title, n.created FROM node n WHERE n.uid = $uid
+ *   ORDER BY n.created DESC LIMIT 0, 10;
  * @endcode
  * one would instead call the Drupal functions:
  * @code
  * $result = db_query_range('SELECT n.nid, n.title, n.created
- *   FROM {node} n WHERE n.uid = :uid', 0, 10, array(':uid' => $uid));
+ *   FROM {node} n WHERE n.uid = :uid
+ *   ORDER BY n.created DESC', 0, 10, array(':uid' => $uid));
  * foreach ($result as $record) {
  *   // Perform operations on $record->title, etc. here.
  * }
@@ -179,7 +182,7 @@
  * concrete implementation of it to support special handling required by that
  * database.
  *
- * @see http://php.net/manual/en/book.pdo.php
+ * @see http://php.net/manual/book.pdo.php
  */
 abstract class DatabaseConnection extends PDO {
 
@@ -293,6 +296,20 @@ abstract class DatabaseConnection extends PDO {
    */
   protected $prefixReplace = array();
 
+  /**
+   * List of escaped database, table, and field names, keyed by unescaped names.
+   *
+   * @var array
+   */
+  protected $escapedNames = array();
+
+  /**
+   * List of escaped aliases names, keyed by unescaped aliases.
+   *
+   * @var array
+   */
+  protected $escapedAliases = array();
+
   function __construct($dsn, $username, $password, $driver_options = array()) {
     // Initialize and prepare the connection prefix.
     $this->setPrefix(isset($this->connectionOptions['prefix']) ? $this->connectionOptions['prefix'] : '');
@@ -623,7 +640,7 @@ abstract class DatabaseConnection extends PDO {
    *   A sanitized version of the query comment string.
    */
   protected function filterComment($comment = '') {
-    return preg_replace('/(\/\*\s*)|(\s*\*\/)/', '', $comment);
+    return strtr($comment, array('*' => ' * '));
   }
 
   /**
@@ -653,7 +670,7 @@ abstract class DatabaseConnection extends PDO {
    * @return DatabaseStatementInterface
    *   This method will return one of: the executed statement, the number of
    *   rows affected by the query (not the number matched), or the generated
-   *   insert IT of the last query, depending on the value of
+   *   insert ID of the last query, depending on the value of
    *   $options['return']. Typically that value will be set by default or a
    *   query builder and should not be set by a user. If there is an error,
    *   this method will return NULL and may throw an exception if
@@ -733,7 +750,7 @@ abstract class DatabaseConnection extends PDO {
     // to expand it out into a comma-delimited set of placeholders.
     foreach (array_filter($args, 'is_array') as $key => $data) {
       $new_keys = array();
-      foreach ($data as $i => $value) {
+      foreach (array_values($data) as $i => $value) {
         // This assumes that there are no other placeholders that use the same
         // name.  For example, if the array placeholder is defined as :example
         // and there is already an :example_2 placeholder, this will generate
@@ -916,11 +933,14 @@ abstract class DatabaseConnection extends PDO {
    * For some database drivers, it may also wrap the table name in
    * database-specific escape characters.
    *
-   * @return
+   * @return string
    *   The sanitized table name string.
    */
   public function escapeTable($table) {
-    return preg_replace('/[^A-Za-z0-9_.]+/', '', $table);
+    if (!isset($this->escapedNames[$table])) {
+      $this->escapedNames[$table] = preg_replace('/[^A-Za-z0-9_.]+/', '', $table);
+    }
+    return $this->escapedNames[$table];
   }
 
   /**
@@ -930,11 +950,14 @@ abstract class DatabaseConnection extends PDO {
    * For some database drivers, it may also wrap the field name in
    * database-specific escape characters.
    *
-   * @return
+   * @return string
    *   The sanitized field name string.
    */
   public function escapeField($field) {
-    return preg_replace('/[^A-Za-z0-9_.]+/', '', $field);
+    if (!isset($this->escapedNames[$field])) {
+      $this->escapedNames[$field] = preg_replace('/[^A-Za-z0-9_.]+/', '', $field);
+    }
+    return $this->escapedNames[$field];
   }
 
   /**
@@ -945,11 +968,14 @@ abstract class DatabaseConnection extends PDO {
    * DatabaseConnection::escapeTable(), this doesn't allow the period (".")
    * because that is not allowed in aliases.
    *
-   * @return
+   * @return string
    *   The sanitized field name string.
    */
   public function escapeAlias($field) {
-    return preg_replace('/[^A-Za-z0-9_]+/', '', $field);
+    if (!isset($this->escapedAliases[$field])) {
+      $this->escapedAliases[$field] = preg_replace('/[^A-Za-z0-9_]+/', '', $field);
+    }
+    return $this->escapedAliases[$field];
   }
 
   /**
@@ -1310,6 +1336,39 @@ abstract class DatabaseConnection extends PDO {
    *   also larger than the $existing_id if one was passed in.
    */
   abstract public function nextId($existing_id = 0);
+
+  /**
+   * Checks whether utf8mb4 support is configurable in settings.php.
+   *
+   * @return bool
+   */
+  public function utf8mb4IsConfigurable() {
+    // Since 4 byte UTF-8 is not supported by default, there is nothing to
+    // configure.
+    return FALSE;
+  }
+
+  /**
+   * Checks whether utf8mb4 support is currently active.
+   *
+   * @return bool
+   */
+  public function utf8mb4IsActive() {
+    // Since 4 byte UTF-8 is not supported by default, there is nothing to
+    // activate.
+    return FALSE;
+  }
+
+  /**
+   * Checks whether utf8mb4 support is available on the current database system.
+   *
+   * @return bool
+   */
+  public function utf8mb4IsSupported() {
+    // By default we assume that the database backend may not support 4 byte
+    // UTF-8.
+    return FALSE;
+  }
 }
 
 /**
@@ -1986,7 +2045,7 @@ interface DatabaseStatementInterface extends Traversable {
   /**
    * Sets the default fetch mode for this statement.
    *
-   * See http://php.net/manual/en/pdo.constants.php for the definition of the
+   * See http://php.net/manual/pdo.constants.php for the definition of the
    * constants used.
    *
    * @param $mode
@@ -2005,7 +2064,7 @@ interface DatabaseStatementInterface extends Traversable {
   /**
    * Fetches the next row from a result set.
    *
-   * See http://php.net/manual/en/pdo.constants.php for the definition of the
+   * See http://php.net/manual/pdo.constants.php for the definition of the
    * constants used.
    *
    * @param $mode
@@ -2380,14 +2439,14 @@ function db_query_range($query, $from, $count, array $args = array(), array $opt
 }
 
 /**
- * Executes a query string and saves the result set to a temporary table.
+ * Executes a SELECT query string and saves the result set to a temporary table.
  *
  * The execution of the query string happens against the active database.
  *
  * @param $query
- *   The prepared statement query to run. Although it will accept both named and
- *   unnamed placeholders, named placeholders are strongly preferred as they are
- *   more self-documenting.
+ *   The prepared SELECT statement query to run. Although it will accept both
+ *   named and unnamed placeholders, named placeholders are strongly preferred
+ *   as they are more self-documenting.
  * @param $args
  *   An array of values to substitute into the query. If the query uses named
  *   placeholders, this is an associative array in any order. If the query uses
@@ -2829,7 +2888,7 @@ function db_drop_table($table) {
  *   will be set to the value of the key in all rows. This is most useful for
  *   creating NOT NULL columns with no default value in existing tables.
  * @param $keys_new
- *   Optional keys and indexes specification to be created on the table along
+ *   (optional) Keys and indexes specification to be created on the table along
  *   with adding the field. The format is the same as a table specification, but
  *   without the 'fields' element. If you are adding a type 'serial' field, you
  *   MUST specify at least one key or index including it in this array. See
@@ -3009,7 +3068,7 @@ function db_drop_index($table, $name) {
  * @param $spec
  *   The field specification for the new field.
  * @param $keys_new
- *   Optional keys and indexes specification to be created on the table along
+ *   (optional) Keys and indexes specification to be created on the table along
  *   with changing the field. The format is the same as a table specification
  *   but without the 'fields' element.
  */

includes/database/mysql/database.inc

@@ -28,6 +28,12 @@ class DatabaseConnection_mysql extends DatabaseConnection {
 
     $this->connectionOptions = $connection_options;
 
+    $charset = 'utf8';
+    // Check if the charset is overridden to utf8mb4 in settings.php.
+    if ($this->utf8mb4IsActive()) {
+      $charset = 'utf8mb4';
+    }
+
     // The DSN should use either a socket or a host/port.
     if (isset($connection_options['unix_socket'])) {
       $dsn = 'mysql:unix_socket=' . $connection_options['unix_socket'];
@@ -36,6 +42,10 @@ class DatabaseConnection_mysql extends DatabaseConnection {
       // Default to TCP connection on port 3306.
       $dsn = 'mysql:host=' . $connection_options['host'] . ';port=' . (empty($connection_options['port']) ? 3306 : $connection_options['port']);
     }
+    // Character set is added to dsn to ensure PDO uses the proper character
+    // set when escaping. This has security implications. See
+    // https://www.drupal.org/node/1201452 for further discussion.
+    $dsn .= ';charset=' . $charset;
     $dsn .= ';dbname=' . $connection_options['database'];
     // Allow PDO options to be overridden.
     $connection_options += array(
@@ -47,6 +57,11 @@ class DatabaseConnection_mysql extends DatabaseConnection {
       // Because MySQL's prepared statements skip the query cache, because it's dumb.
       PDO::ATTR_EMULATE_PREPARES => TRUE,
     );
+    if (defined('PDO::MYSQL_ATTR_MULTI_STATEMENTS')) {
+      // An added connection option in PHP 5.5.21+ to optionally limit SQL to a
+      // single statement like mysqli.
+      $connection_options['pdo'] += array(PDO::MYSQL_ATTR_MULTI_STATEMENTS => FALSE);
+    }
 
     parent::__construct($dsn, $connection_options['username'], $connection_options['password'], $connection_options['pdo']);
 
@@ -54,10 +69,10 @@ class DatabaseConnection_mysql extends DatabaseConnection {
     // certain one has been set; otherwise, MySQL defaults to 'utf8_general_ci'
     // for UTF-8.
     if (!empty($connection_options['collation'])) {
-      $this->exec('SET NAMES utf8 COLLATE ' . $connection_options['collation']);
+      $this->exec('SET NAMES ' . $charset . ' COLLATE ' . $connection_options['collation']);
     }
     else {
-      $this->exec('SET NAMES utf8');
+      $this->exec('SET NAMES ' . $charset);
     }
 
     // Set MySQL init_commands if not already defined.  Default Drupal's MySQL
@@ -72,10 +87,12 @@ class DatabaseConnection_mysql extends DatabaseConnection {
       'init_commands' => array(),
     );
     $connection_options['init_commands'] += array(
-      'sql_mode' => "SET sql_mode = 'ANSI,STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER'",
+      'sql_mode' => "SET sql_mode = 'REAL_AS_FLOAT,PIPES_AS_CONCAT,ANSI_QUOTES,IGNORE_SPACE,STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER'",
     );
-    // Set connection options.
-    $this->exec(implode('; ', $connection_options['init_commands']));
+    // Execute initial commands.
+    foreach ($connection_options['init_commands'] as $sql) {
+      $this->exec($sql);
+    }
   }
 
   public function __destruct() {
@@ -90,7 +107,7 @@ class DatabaseConnection_mysql extends DatabaseConnection {
 
   public function queryTemporary($query, array $args = array(), array $options = array()) {
     $tablename = $this->generateTemporaryTableName();
-    $this->query(preg_replace('/^SELECT/i', 'CREATE TEMPORARY TABLE {' . $tablename . '} Engine=MEMORY SELECT', $query), $args, $options);
+    $this->query('CREATE TEMPORARY TABLE {' . $tablename . '} Engine=MEMORY ' . $query, $args, $options);
     return $tablename;
   }
 
@@ -195,6 +212,42 @@ class DatabaseConnection_mysql extends DatabaseConnection {
       }
     }
   }
+
+  public function utf8mb4IsConfigurable() {
+    return TRUE;
+  }
+
+  public function utf8mb4IsActive() {
+    return isset($this->connectionOptions['charset']) && $this->connectionOptions['charset'] === 'utf8mb4';
+  }
+
+  public function utf8mb4IsSupported() {
+    // Ensure that the MySQL driver supports utf8mb4 encoding.
+    $version = $this->getAttribute(PDO::ATTR_CLIENT_VERSION);
+    if (strpos($version, 'mysqlnd') !== FALSE) {
+      // The mysqlnd driver supports utf8mb4 starting at version 5.0.9.
+      $version = preg_replace('/^\D+([\d.]+).*/', '$1', $version);
+      if (version_compare($version, '5.0.9', '<')) {
+        return FALSE;
+      }
+    }
+    else {
+      // The libmysqlclient driver supports utf8mb4 starting at version 5.5.3.
+      if (version_compare($version, '5.5.3', '<')) {
+        return FALSE;
+      }
+    }
+
+    // Ensure that the MySQL server supports large prefixes and utf8mb4.
+    try {
+      $this->query("CREATE TABLE {drupal_utf8mb4_test} (id VARCHAR(255), PRIMARY KEY(id(255))) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci ROW_FORMAT=DYNAMIC ENGINE=INNODB");
+    }
+    catch (Exception $e) {
+      return FALSE;
+    }
+    $this->query("DROP TABLE {drupal_utf8mb4_test}");
+    return TRUE;
+  }
 }
 
 

includes/database/mysql/query.inc

@@ -51,7 +51,8 @@ class InsertQuery_mysql extends InsertQuery {
     // If we're selecting from a SelectQuery, finish building the query and
     // pass it back, as any remaining options are irrelevant.
     if (!empty($this->fromQuery)) {
-      return $comments . 'INSERT INTO {' . $this->table . '} (' . implode(', ', $insert_fields) . ') ' . $this->fromQuery;
+      $insert_fields_string = $insert_fields ? ' (' . implode(', ', $insert_fields) . ') ' : ' ';
+      return $comments . 'INSERT INTO {' . $this->table . '}' . $insert_fields_string . $this->fromQuery;
     }
 
     $query = $comments . 'INSERT INTO {' . $this->table . '} (' . implode(', ', $insert_fields) . ') VALUES ';
@@ -86,21 +87,7 @@ class InsertQuery_mysql extends InsertQuery {
   }
 }
 
-class TruncateQuery_mysql extends TruncateQuery {
-  public function __toString() {
-    // TRUNCATE is actually a DDL statement on MySQL, and DDL statements are
-    // not transactional, and result in an implicit COMMIT. When we are in a
-    // transaction, fallback to the slower, but transactional, DELETE.
-    if ($this->connection->inTransaction()) {
-      // Create a comment string to prepend to the query.
-      $comments = $this->connection->makeComment($this->comments);
-      return $comments . 'DELETE FROM {' . $this->connection->escapeTable($this->table) . '}';
-    }
-    else {
-      return parent::__toString();
-    }
-  }
-}
+class TruncateQuery_mysql extends TruncateQuery { }
 
 /**
  * @} End of "addtogroup database".

includes/database/mysql/schema.inc

@@ -39,8 +39,8 @@ class DatabaseSchema_mysql extends DatabaseSchema {
       $info['table'] = substr($table, ++$pos);
     }
     else {
-      $db_info = Database::getConnectionInfo();
-      $info['database'] = $db_info['default']['database'];
+      $db_info = $this->connection->getConnectionOptions();
+      $info['database'] = $db_info['database'];
       $info['table'] = $table;
     }
     return $info;
@@ -81,7 +81,8 @@ class DatabaseSchema_mysql extends DatabaseSchema {
     // Provide defaults if needed.
     $table += array(
       'mysql_engine' => 'InnoDB',
-      'mysql_character_set' => 'utf8',
+      // Allow the default charset to be overridden in settings.php.
+      'mysql_character_set' => $this->connection->utf8mb4IsActive() ? 'utf8mb4' : 'utf8',
     );
 
     $sql = "CREATE TABLE {" . $name . "} (\n";
@@ -109,6 +110,13 @@ class DatabaseSchema_mysql extends DatabaseSchema {
       $sql .= ' COLLATE ' . $info['collation'];
     }
 
+    // The row format needs to be either DYNAMIC or COMPRESSED in order to allow
+    // for the innodb_large_prefix setting to take effect, see
+    // https://dev.mysql.com/doc/refman/5.6/en/create-table.html
+    if ($this->connection->utf8mb4IsActive()) {
+      $sql .= ' ROW_FORMAT=DYNAMIC';
+    }
+
     // Add table comment.
     if (!empty($table['description'])) {
       $sql .= ' COMMENT ' . $this->prepareComment($table['description'], self::COMMENT_MAX_TABLE);
@@ -301,10 +309,10 @@ class DatabaseSchema_mysql extends DatabaseSchema {
 
   public function renameTable($table, $new_name) {
     if (!$this->tableExists($table)) {
-      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot rename %table to %table_new: table %table doesn't exist.", array('%table' => $table, '%table_new' => $new_name)));
+      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot rename @table to @table_new: table @table doesn't exist.", array('@table' => $table, '@table_new' => $new_name)));
     }
     if ($this->tableExists($new_name)) {
-      throw new DatabaseSchemaObjectExistsException(t("Cannot rename %table to %table_new: table %table_new already exists.", array('%table' => $table, '%table_new' => $new_name)));
+      throw new DatabaseSchemaObjectExistsException(t("Cannot rename @table to @table_new: table @table_new already exists.", array('@table' => $table, '@table_new' => $new_name)));
     }
 
     $info = $this->getPrefixInfo($new_name);
@@ -322,10 +330,10 @@ class DatabaseSchema_mysql extends DatabaseSchema {
 
   public function addField($table, $field, $spec, $keys_new = array()) {
     if (!$this->tableExists($table)) {
-      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot add field %table.%field: table doesn't exist.", array('%field' => $field, '%table' => $table)));
+      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot add field @table.@field: table doesn't exist.", array('@field' => $field, '@table' => $table)));
     }
     if ($this->fieldExists($table, $field)) {
-      throw new DatabaseSchemaObjectExistsException(t("Cannot add field %table.%field: field already exists.", array('%field' => $field, '%table' => $table)));
+      throw new DatabaseSchemaObjectExistsException(t("Cannot add field @table.@field: field already exists.", array('@field' => $field, '@table' => $table)));
     }
 
     $fixnull = FALSE;
@@ -361,7 +369,7 @@ class DatabaseSchema_mysql extends DatabaseSchema {
 
   public function fieldSetDefault($table, $field, $default) {
     if (!$this->fieldExists($table, $field)) {
-      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot set default value of field %table.%field: field doesn't exist.", array('%table' => $table, '%field' => $field)));
+      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot set default value of field @table.@field: field doesn't exist.", array('@table' => $table, '@field' => $field)));
     }
 
     if (!isset($default)) {
@@ -376,7 +384,7 @@ class DatabaseSchema_mysql extends DatabaseSchema {
 
   public function fieldSetNoDefault($table, $field) {
     if (!$this->fieldExists($table, $field)) {
-      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot remove default value of field %table.%field: field doesn't exist.", array('%table' => $table, '%field' => $field)));
+      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot remove default value of field @table.@field: field doesn't exist.", array('@table' => $table, '@field' => $field)));
     }
 
     $this->connection->query('ALTER TABLE {' . $table . '} ALTER COLUMN `' . $field . '` DROP DEFAULT');
@@ -391,10 +399,10 @@ class DatabaseSchema_mysql extends DatabaseSchema {
 
   public function addPrimaryKey($table, $fields) {
     if (!$this->tableExists($table)) {
-      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot add primary key to table %table: table doesn't exist.", array('%table' => $table)));
+      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot add primary key to table @table: table doesn't exist.", array('@table' => $table)));
     }
     if ($this->indexExists($table, 'PRIMARY')) {
-      throw new DatabaseSchemaObjectExistsException(t("Cannot add primary key to table %table: primary key already exists.", array('%table' => $table)));
+      throw new DatabaseSchemaObjectExistsException(t("Cannot add primary key to table @table: primary key already exists.", array('@table' => $table)));
     }
 
     $this->connection->query('ALTER TABLE {' . $table . '} ADD PRIMARY KEY (' . $this->createKeySql($fields) . ')');
@@ -411,10 +419,10 @@ class DatabaseSchema_mysql extends DatabaseSchema {
 
   public function addUniqueKey($table, $name, $fields) {
     if (!$this->tableExists($table)) {
-      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot add unique key %name to table %table: table doesn't exist.", array('%table' => $table, '%name' => $name)));
+      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot add unique key @name to table @table: table doesn't exist.", array('@table' => $table, '@name' => $name)));
     }
     if ($this->indexExists($table, $name)) {
-      throw new DatabaseSchemaObjectExistsException(t("Cannot add unique key %name to table %table: unique key already exists.", array('%table' => $table, '%name' => $name)));
+      throw new DatabaseSchemaObjectExistsException(t("Cannot add unique key @name to table @table: unique key already exists.", array('@table' => $table, '@name' => $name)));
     }
 
     $this->connection->query('ALTER TABLE {' . $table . '} ADD UNIQUE KEY `' . $name . '` (' . $this->createKeySql($fields) . ')');
@@ -431,10 +439,10 @@ class DatabaseSchema_mysql extends DatabaseSchema {
 
   public function addIndex($table, $name, $fields) {
     if (!$this->tableExists($table)) {
-      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot add index %name to table %table: table doesn't exist.", array('%table' => $table, '%name' => $name)));
+      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot add index @name to table @table: table doesn't exist.", array('@table' => $table, '@name' => $name)));
     }
     if ($this->indexExists($table, $name)) {
-      throw new DatabaseSchemaObjectExistsException(t("Cannot add index %name to table %table: index already exists.", array('%table' => $table, '%name' => $name)));
+      throw new DatabaseSchemaObjectExistsException(t("Cannot add index @name to table @table: index already exists.", array('@table' => $table, '@name' => $name)));
     }
 
     $this->connection->query('ALTER TABLE {' . $table . '} ADD INDEX `' . $name . '` (' . $this->createKeySql($fields) . ')');
@@ -451,10 +459,10 @@ class DatabaseSchema_mysql extends DatabaseSchema {
 
   public function changeField($table, $field, $field_new, $spec, $keys_new = array()) {
     if (!$this->fieldExists($table, $field)) {
-      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot change the definition of field %table.%name: field doesn't exist.", array('%table' => $table, '%name' => $field)));
+      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot change the definition of field @table.@name: field doesn't exist.", array('@table' => $table, '@name' => $field)));
     }
     if (($field != $field_new) && $this->fieldExists($table, $field_new)) {
-      throw new DatabaseSchemaObjectExistsException(t("Cannot rename field %table.%name to %name_new: target field already exists.", array('%table' => $table, '%name' => $field, '%name_new' => $field_new)));
+      throw new DatabaseSchemaObjectExistsException(t("Cannot rename field @table.@name to @name_new: target field already exists.", array('@table' => $table, '@name' => $field, '@name_new' => $field_new)));
     }
 
     $sql = 'ALTER TABLE {' . $table . '} CHANGE `' . $field . '` ' . $this->createFieldSql($field_new, $this->processField($spec));

includes/database/pgsql/database.inc

@@ -11,7 +11,7 @@
  */
 
 /**
- * The name by which to obtain a lock for retrive the next insert id.
+ * The name by which to obtain a lock for retrieving the next insert id.
  */
 define('POSTGRESQL_NEXTID_LOCK', 1000);
 
@@ -55,7 +55,7 @@ class DatabaseConnection_pgsql extends DatabaseConnection {
     $connection_options['pdo'] += array(
       // Prepared statements are most effective for performance when queries
       // are recycled (used several times). However, if they are not re-used,
-      // prepared statements become ineffecient. Since most of Drupal's
+      // prepared statements become inefficient. Since most of Drupal's
       // prepared queries are not re-used, it should be faster to emulate
       // the preparation than to actually ready statements for re-use. If in
       // doubt, reset to FALSE and measure performance.
@@ -146,7 +146,7 @@ class DatabaseConnection_pgsql extends DatabaseConnection {
 
   public function queryTemporary($query, array $args = array(), array $options = array()) {
     $tablename = $this->generateTemporaryTableName();
-    $this->query(preg_replace('/^SELECT/i', 'CREATE TEMPORARY TABLE {' . $tablename . '} AS SELECT', $query), $args, $options);
+    $this->query('CREATE TEMPORARY TABLE {' . $tablename . '} AS ' . $query, $args, $options);
     return $tablename;
   }
 
@@ -175,14 +175,14 @@ class DatabaseConnection_pgsql extends DatabaseConnection {
   }
 
   /**
-   * Retrive a the next id in a sequence.
+   * Retrieve the next id in a sequence.
    *
    * PostgreSQL has built in sequences. We'll use these instead of inserting
    * and updating a sequences table.
    */
   public function nextId($existing = 0) {
 
-    // Retrive the name of the sequence. This information cannot be cached
+    // Retrieve the name of the sequence. This information cannot be cached
     // because the prefix may change, for example, like it does in simpletests.
     $sequence_name = $this->makeSequenceName('sequences', 'value');
 
@@ -194,7 +194,7 @@ class DatabaseConnection_pgsql extends DatabaseConnection {
     }
 
     // PostgreSQL advisory locks are simply locks to be used by an
-    // application such as Drupal. This will prevent other Drupal proccesses
+    // application such as Drupal. This will prevent other Drupal processes
     // from altering the sequence while we are.
     $this->query("SELECT pg_advisory_lock(" . POSTGRESQL_NEXTID_LOCK . ")");
 
@@ -209,13 +209,21 @@ class DatabaseConnection_pgsql extends DatabaseConnection {
     // Reset the sequence to a higher value than the existing id.
     $this->query("ALTER SEQUENCE " . $sequence_name . " RESTART WITH " . ($existing + 1));
 
-    // Retrive the next id. We know this will be as high as we want it.
+    // Retrieve the next id. We know this will be as high as we want it.
     $id = $this->query("SELECT nextval('" . $sequence_name . "')")->fetchField();
 
     $this->query("SELECT pg_advisory_unlock(" . POSTGRESQL_NEXTID_LOCK . ")");
 
     return $id;
   }
+
+  public function utf8mb4IsActive() {
+    return TRUE;
+  }
+
+  public function utf8mb4IsSupported() {
+    return TRUE;
+  }
 }
 
 /**

includes/database/pgsql/install.inc

@@ -165,7 +165,7 @@ class DatabaseTasks_pgsql extends DatabaseTasks {
         LANGUAGE \'sql\''
       );
 
-      // Using || to concatenate in Drupal is not recommeneded because there are
+      // Using || to concatenate in Drupal is not recommended because there are
       // database drivers for Drupal that do not support the syntax, however
       // they do support CONCAT(item1, item2) which we can replicate in
       // PostgreSQL. PostgreSQL requires the function to be defined for each

includes/database/pgsql/query.inc

@@ -112,7 +112,8 @@ class InsertQuery_pgsql extends InsertQuery {
     // If we're selecting from a SelectQuery, finish building the query and
     // pass it back, as any remaining options are irrelevant.
     if (!empty($this->fromQuery)) {
-      return $comments . 'INSERT INTO {' . $this->table . '} (' . implode(', ', $insert_fields) . ') ' . $this->fromQuery;
+      $insert_fields_string = $insert_fields ? ' (' . implode(', ', $insert_fields) . ') ' : ' ';
+      return $comments . 'INSERT INTO {' . $this->table . '}' . $insert_fields_string . $this->fromQuery;
     }
 
     $query = $comments . 'INSERT INTO {' . $this->table . '} (' . implode(', ', $insert_fields) . ') VALUES ';

includes/database/pgsql/schema.inc

@@ -314,10 +314,10 @@ class DatabaseSchema_pgsql extends DatabaseSchema {
 
   function renameTable($table, $new_name) {
     if (!$this->tableExists($table)) {
-      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot rename %table to %table_new: table %table doesn't exist.", array('%table' => $table, '%table_new' => $new_name)));
+      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot rename @table to @table_new: table @table doesn't exist.", array('@table' => $table, '@table_new' => $new_name)));
     }
     if ($this->tableExists($new_name)) {
-      throw new DatabaseSchemaObjectExistsException(t("Cannot rename %table to %table_new: table %table_new already exists.", array('%table' => $table, '%table_new' => $new_name)));
+      throw new DatabaseSchemaObjectExistsException(t("Cannot rename @table to @table_new: table @table_new already exists.", array('@table' => $table, '@table_new' => $new_name)));
     }
 
     // Get the schema and tablename for the old table.
@@ -351,10 +351,10 @@ class DatabaseSchema_pgsql extends DatabaseSchema {
 
   public function addField($table, $field, $spec, $new_keys = array()) {
     if (!$this->tableExists($table)) {
-      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot add field %table.%field: table doesn't exist.", array('%field' => $field, '%table' => $table)));
+      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot add field @table.@field: table doesn't exist.", array('@field' => $field, '@table' => $table)));
     }
     if ($this->fieldExists($table, $field)) {
-      throw new DatabaseSchemaObjectExistsException(t("Cannot add field %table.%field: field already exists.", array('%field' => $field, '%table' => $table)));
+      throw new DatabaseSchemaObjectExistsException(t("Cannot add field @table.@field: field already exists.", array('@field' => $field, '@table' => $table)));
     }
 
     $fixnull = FALSE;
@@ -393,7 +393,7 @@ class DatabaseSchema_pgsql extends DatabaseSchema {
 
   public function fieldSetDefault($table, $field, $default) {
     if (!$this->fieldExists($table, $field)) {
-      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot set default value of field %table.%field: field doesn't exist.", array('%table' => $table, '%field' => $field)));
+      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot set default value of field @table.@field: field doesn't exist.", array('@table' => $table, '@field' => $field)));
     }
 
     if (!isset($default)) {
@@ -408,7 +408,7 @@ class DatabaseSchema_pgsql extends DatabaseSchema {
 
   public function fieldSetNoDefault($table, $field) {
     if (!$this->fieldExists($table, $field)) {
-      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot remove default value of field %table.%field: field doesn't exist.", array('%table' => $table, '%field' => $field)));
+      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot remove default value of field @table.@field: field doesn't exist.", array('@table' => $table, '@field' => $field)));
     }
 
     $this->connection->query('ALTER TABLE {' . $table . '} ALTER COLUMN "' . $field . '" DROP DEFAULT');
@@ -435,10 +435,10 @@ class DatabaseSchema_pgsql extends DatabaseSchema {
 
   public function addPrimaryKey($table, $fields) {
     if (!$this->tableExists($table)) {
-      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot add primary key to table %table: table doesn't exist.", array('%table' => $table)));
+      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot add primary key to table @table: table doesn't exist.", array('@table' => $table)));
     }
     if ($this->constraintExists($table, 'pkey')) {
-      throw new DatabaseSchemaObjectExistsException(t("Cannot add primary key to table %table: primary key already exists.", array('%table' => $table)));
+      throw new DatabaseSchemaObjectExistsException(t("Cannot add primary key to table @table: primary key already exists.", array('@table' => $table)));
     }
 
     $this->connection->query('ALTER TABLE {' . $table . '} ADD PRIMARY KEY (' . implode(',', $fields) . ')');
@@ -455,10 +455,10 @@ class DatabaseSchema_pgsql extends DatabaseSchema {
 
   function addUniqueKey($table, $name, $fields) {
     if (!$this->tableExists($table)) {
-      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot add unique key %name to table %table: table doesn't exist.", array('%table' => $table, '%name' => $name)));
+      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot add unique key @name to table @table: table doesn't exist.", array('@table' => $table, '@name' => $name)));
     }
     if ($this->constraintExists($table, $name . '_key')) {
-      throw new DatabaseSchemaObjectExistsException(t("Cannot add unique key %name to table %table: unique key already exists.", array('%table' => $table, '%name' => $name)));
+      throw new DatabaseSchemaObjectExistsException(t("Cannot add unique key @name to table @table: unique key already exists.", array('@table' => $table, '@name' => $name)));
     }
 
     $this->connection->query('ALTER TABLE {' . $table . '} ADD CONSTRAINT "' . $this->prefixNonTable($table, $name, 'key') . '" UNIQUE (' . implode(',', $fields) . ')');
@@ -475,10 +475,10 @@ class DatabaseSchema_pgsql extends DatabaseSchema {
 
   public function addIndex($table, $name, $fields) {
     if (!$this->tableExists($table)) {
-      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot add index %name to table %table: table doesn't exist.", array('%table' => $table, '%name' => $name)));
+      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot add index @name to table @table: table doesn't exist.", array('@table' => $table, '@name' => $name)));
     }
     if ($this->indexExists($table, $name)) {
-      throw new DatabaseSchemaObjectExistsException(t("Cannot add index %name to table %table: index already exists.", array('%table' => $table, '%name' => $name)));
+      throw new DatabaseSchemaObjectExistsException(t("Cannot add index @name to table @table: index already exists.", array('@table' => $table, '@name' => $name)));
     }
 
     $this->connection->query($this->_createIndexSql($table, $name, $fields));
@@ -495,10 +495,10 @@ class DatabaseSchema_pgsql extends DatabaseSchema {
 
   public function changeField($table, $field, $field_new, $spec, $new_keys = array()) {
     if (!$this->fieldExists($table, $field)) {
-      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot change the definition of field %table.%name: field doesn't exist.", array('%table' => $table, '%name' => $field)));
+      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot change the definition of field @table.@name: field doesn't exist.", array('@table' => $table, '@name' => $field)));
     }
     if (($field != $field_new) && $this->fieldExists($table, $field_new)) {
-      throw new DatabaseSchemaObjectExistsException(t("Cannot rename field %table.%name to %name_new: target field already exists.", array('%table' => $table, '%name' => $field, '%name_new' => $field_new)));
+      throw new DatabaseSchemaObjectExistsException(t("Cannot rename field @table.@name to @name_new: target field already exists.", array('@table' => $table, '@name' => $field, '@name_new' => $field_new)));
     }
 
     $spec = $this->processField($spec);

includes/database/pgsql/select.inc

@@ -80,7 +80,7 @@ class SelectQuery_pgsql extends SelectQuery {
     }
 
     // If a table loads all fields, it can not be added again. It would
-    // result in an ambigious alias error because that field would be loaded
+    // result in an ambiguous alias error because that field would be loaded
     // twice: Once through table_alias.* and once directly. If the field
     // actually belongs to a different table, it must be added manually.
     foreach ($this->tables as $table) {
@@ -90,7 +90,7 @@ class SelectQuery_pgsql extends SelectQuery {
     }
 
     // If $field contains an characters which are not allowed in a field name
-    // it is considered an expression, these can't be handeld automatically
+    // it is considered an expression, these can't be handled automatically
     // either.
     if ($this->connection->escapeField($field) != $field) {
       return $return;

includes/database/query.inc

@@ -83,7 +83,7 @@ interface QueryConditionInterface {
 
   /**
    * Sets a condition that the specified subquery returns values.
-   * 
+   *
    * @param SelectQueryInterface $select
    *   The subquery that must contain results.
    *
@@ -91,10 +91,10 @@ interface QueryConditionInterface {
    *   The called object.
    */
   public function exists(SelectQueryInterface $select);
-  
+
   /**
    * Sets a condition that the specified subquery returns no values.
-   * 
+   *
    * @param SelectQueryInterface $select
    *   The subquery that must not contain results.
    *
@@ -102,7 +102,7 @@ interface QueryConditionInterface {
    *   The called object.
    */
   public function notExists(SelectQueryInterface $select);
-  
+
   /**
    * Gets a complete list of all conditions in this conditional clause.
    *
@@ -283,14 +283,14 @@ abstract class Query implements QueryPlaceholderInterface {
 
   /**
    * The target of the connection object.
-   * 
+   *
    * @var string
    */
   protected $connectionTarget;
 
   /**
    * The key of the connection object.
-   * 
+   *
    * @var string
    */
   protected $connectionKey;
@@ -710,10 +710,11 @@ class InsertQuery extends Query {
       // first call to fields() does have an effect.
       $this->fields(array_merge(array_keys($this->fromQuery->getFields()), array_keys($this->fromQuery->getExpressions())));
     }
-
-    // Don't execute query without fields.
-    if (count($this->insertFields) + count($this->defaultFields) == 0) {
-      throw new NoFieldsException('There are no fields available to insert with.');
+    else {
+      // Don't execute query without fields.
+      if (count($this->insertFields) + count($this->defaultFields) == 0) {
+        throw new NoFieldsException('There are no fields available to insert with.');
+      }
     }
 
     // If no values have been added, silently ignore this query. This can happen
@@ -804,7 +805,7 @@ class DeleteQuery extends Query implements QueryConditionInterface {
     $this->condition->notExists($select);
     return $this;
   }
-  
+
   /**
    * Implements QueryConditionInterface::conditions().
    */
@@ -844,8 +845,8 @@ class DeleteQuery extends Query implements QueryConditionInterface {
   /**
    * Executes the DELETE query.
    *
-   * @return
-   *   The return value is dependent on the database connection.
+   * @return int
+   *   The number of rows affected by the delete query.
    */
   public function execute() {
     $values = array();
@@ -942,7 +943,17 @@ class TruncateQuery extends Query {
     // Create a sanitized comment string to prepend to the query.
     $comments = $this->connection->makeComment($this->comments);
 
-    return $comments . 'TRUNCATE {' . $this->connection->escapeTable($this->table) . '} ';
+    // In most cases, TRUNCATE is not a transaction safe statement as it is a
+    // DDL statement which results in an implicit COMMIT. When we are in a
+    // transaction, fallback to the slower, but transactional, DELETE.
+    // PostgreSQL also locks the entire table for a TRUNCATE strongly reducing
+    // the concurrency with other transactions.
+    if ($this->connection->inTransaction()) {
+      return $comments . 'DELETE FROM {' . $this->connection->escapeTable($this->table) . '}';
+    }
+    else {
+      return $comments . 'TRUNCATE {' . $this->connection->escapeTable($this->table) . '} ';
+    }
   }
 }
 
@@ -1053,7 +1064,7 @@ class UpdateQuery extends Query implements QueryConditionInterface {
     $this->condition->notExists($select);
     return $this;
   }
-  
+
   /**
    * Implements QueryConditionInterface::conditions().
    */
@@ -1231,7 +1242,7 @@ class UpdateQuery extends Query implements QueryConditionInterface {
  * MergeQuery::updateFields() and MergeQuery::insertFields() needs to be called
  * instead. MergeQuery::fields() can also be called which calls both of these
  * methods as the common case is to use the same column-value pairs for both
- * INSERT and UPDATE. However, this is not mandatory. Another convinient
+ * INSERT and UPDATE. However, this is not mandatory. Another convenient
  * wrapper is MergeQuery::key() which adds the same column-value pairs to the
  * condition and the INSERT query part.
  *
@@ -1545,7 +1556,7 @@ class MergeQuery extends Query implements QueryConditionInterface {
     $this->condition->notExists($select);
     return $this;
   }
-  
+
   /**
    * Implements QueryConditionInterface::conditions().
    */
@@ -1595,55 +1606,43 @@ class MergeQuery extends Query implements QueryConditionInterface {
   }
 
   public function execute() {
-    // Wrap multiple queries in a transaction, if the database supports it.
-    $transaction = $this->connection->startTransaction();
-    try {
-      if (!count($this->condition)) {
-        throw new InvalidMergeQueryException(t('Invalid merge query: no conditions'));
-      }
-      $select = $this->connection->select($this->conditionTable)
-        ->condition($this->condition)
-        ->forUpdate();
-      $select->addExpression('1');
-      if (!$select->execute()->fetchField()) {
-        try {
-          $insert = $this->connection->insert($this->table)->fields($this->insertFields);
-          if ($this->defaultFields) {
-            $insert->useDefaults($this->defaultFields);
-          }
-          $insert->execute();
-          return MergeQuery::STATUS_INSERT;
-        }
-        catch (Exception $e) {
-          // The insert query failed, maybe it's because a racing insert query
-          // beat us in inserting the same row. Retry the select query, if it
-          // returns a row, ignore the error and continue with the update
-          // query below.
-          if (!$select->execute()->fetchField()) {
-            throw $e;
-          }
+    if (!count($this->condition)) {
+      throw new InvalidMergeQueryException(t('Invalid merge query: no conditions'));
+    }
+    $select = $this->connection->select($this->conditionTable)
+      ->condition($this->condition);
+    $select->addExpression('1');
+    if (!$select->execute()->fetchField()) {
+      try {
+        $insert = $this->connection->insert($this->table)->fields($this->insertFields);
+        if ($this->defaultFields) {
+          $insert->useDefaults($this->defaultFields);
         }
+        $insert->execute();
+        return self::STATUS_INSERT;
       }
-      if ($this->needsUpdate) {
-        $update = $this->connection->update($this->table)
-          ->fields($this->updateFields)
-          ->condition($this->condition);
-        if ($this->expressionFields) {
-          foreach ($this->expressionFields as $field => $data) {
-            $update->expression($field, $data['expression'], $data['arguments']);
-          }
+      catch (Exception $e) {
+        // The insert query failed, maybe it's because a racing insert query
+        // beat us in inserting the same row. Retry the select query, if it
+        // returns a row, ignore the error and continue with the update
+        // query below.
+        if (!$select->execute()->fetchField()) {
+          throw $e;
         }
-        $update->execute();
-        return MergeQuery::STATUS_UPDATE;
       }
     }
-    catch (Exception $e) {
-      // Something really wrong happened here, bubble up the exception to the
-      // caller.
-      $transaction->rollback();
-      throw $e;
-    }
-    // Transaction commits here where $transaction looses scope.
+    if ($this->needsUpdate) {
+      $update = $this->connection->update($this->table)
+        ->fields($this->updateFields)
+        ->condition($this->condition);
+      if ($this->expressionFields) {
+        foreach ($this->expressionFields as $field => $data) {
+          $update->expression($field, $data['expression'], $data['arguments']);
+        }
+      }
+      $update->execute();
+      return self::STATUS_UPDATE;
+     }
   }
 }
 
@@ -1695,7 +1694,7 @@ class DatabaseCondition implements QueryConditionInterface, Countable {
    * Implements Countable::count().
    *
    * Returns the size of this conditional. The size of the conditional is the
-   * size of its conditional array minus one, because one element is the the
+   * size of its conditional array minus one, because one element is the
    * conjunction.
    */
   public function count() {
@@ -1762,14 +1761,14 @@ class DatabaseCondition implements QueryConditionInterface, Countable {
   public function exists(SelectQueryInterface $select) {
     return $this->condition('', $select, 'EXISTS');
   }
-  
+
   /**
    * Implements QueryConditionInterface::notExists().
    */
   public function notExists(SelectQueryInterface $select) {
     return $this->condition('', $select, 'NOT EXISTS');
   }
-  
+
   /**
    * Implements QueryConditionInterface::conditions().
    */

includes/database/schema.inc

@@ -92,7 +92,8 @@ require_once dirname(__FILE__) . '/query.inc';
  *    specification). Each specification is an array containing the name of
  *    the referenced table ('table'), and an array of column mappings
  *    ('columns'). Column mappings are defined by key pairs ('source_column' =>
- *    'referenced_column').
+ *    'referenced_column'). This key is for documentation purposes only; foreign
+ *    keys are not created in the database, nor are they enforced by Drupal.
  *  - 'indexes':  An associative array of indexes ('indexname' =>
  *    specification). Each specification is an array of one or more
  *    key column specifiers (see below) that form an index on the
@@ -144,6 +145,8 @@ require_once dirname(__FILE__) . '/query.inc';
  *   'unique keys' => array(
  *     'vid' => array('vid'),
  *   ),
+ *   // For documentation purposes only; foreign keys are not created in the
+ *   // database.
  *   'foreign keys' => array(
  *     'node_revision' => array(
  *       'table' => 'node_revision',
@@ -161,6 +164,9 @@ require_once dirname(__FILE__) . '/query.inc';
  * @see drupal_install_schema()
  */
 
+/**
+ * Base class for database schema definitions.
+ */
 abstract class DatabaseSchema implements QueryPlaceholderInterface {
 
   protected $connection;
@@ -288,7 +294,7 @@ abstract class DatabaseSchema implements QueryPlaceholderInterface {
   protected function buildTableNameCondition($table_name, $operator = '=', $add_prefix = TRUE) {
     $info = $this->connection->getConnectionOptions();
 
-    // Retrive the table name and schema
+    // Retrieve the table name and schema
     $table_info = $this->getPrefixInfo($table_name, $add_prefix);
 
     $condition = new DatabaseCondition('AND');
@@ -416,7 +422,7 @@ abstract class DatabaseSchema implements QueryPlaceholderInterface {
    *   This is most useful for creating NOT NULL columns with no default
    *   value in existing tables.
    * @param $keys_new
-   *   Optional keys and indexes specification to be created on the
+   *   (optional) Keys and indexes specification to be created on the
    *   table along with adding the field. The format is the same as a
    *   table specification but without the 'fields' element. If you are
    *   adding a type 'serial' field, you MUST specify at least one key
@@ -630,7 +636,7 @@ abstract class DatabaseSchema implements QueryPlaceholderInterface {
    * @param $spec
    *   The field specification for the new field.
    * @param $keys_new
-   *   Optional keys and indexes specification to be created on the
+   *   (optional) Keys and indexes specification to be created on the
    *   table along with changing the field. The format is the same as a
    *   table specification but without the 'fields' element.
    *
@@ -654,7 +660,7 @@ abstract class DatabaseSchema implements QueryPlaceholderInterface {
    */
   public function createTable($name, $table) {
     if ($this->tableExists($name)) {
-      throw new DatabaseSchemaObjectExistsException(t('Table %name already exists.', array('%name' => $name)));
+      throw new DatabaseSchemaObjectExistsException(t('Table @name already exists.', array('@name' => $name)));
     }
     $statements = $this->createTableSql($name, $table);
     foreach ($statements as $statement) {

includes/database/select.inc

@@ -377,7 +377,8 @@ interface SelectQueryInterface extends QueryConditionInterface, QueryAlterableIn
    * @param $field
    *   The field on which to order.
    * @param $direction
-   *   The direction to sort. Legal values are "ASC" and "DESC".
+   *   The direction to sort. Legal values are "ASC" and "DESC". Any other value
+   *   will be converted to "ASC".
    * @return SelectQueryInterface
    *   The called object.
    */
@@ -596,7 +597,7 @@ class SelectQueryExtender implements SelectQueryInterface {
 
   public function hasAnyTag() {
     $args = func_get_args();
-    return call_user_func_array(array($this->query, 'hasAnyTags'), $args);
+    return call_user_func_array(array($this->query, 'hasAnyTag'), $args);
   }
 
   public function addMetaData($key, $object) {
@@ -1230,6 +1231,21 @@ class SelectQuery extends Query implements SelectQueryInterface {
 
     // Modules may alter all queries or only those having a particular tag.
     if (isset($this->alterTags)) {
+      // Many contrib modules assume that query tags used for access-checking
+      // purposes follow the pattern $entity_type . '_access'. But this is
+      // not the case for taxonomy terms, since core used to add term_access
+      // instead of taxonomy_term_access to its queries. Provide backwards
+      // compatibility by adding both tags here instead of attempting to fix
+      // all contrib modules in a coordinated effort.
+      // TODO:
+      // - Extract this mechanism into a hook as part of a public (non-security)
+      //   issue.
+      // - Emit E_USER_DEPRECATED if term_access is used.
+      //   https://www.drupal.org/node/2575081
+      $term_access_tags = array('term_access' => 1, 'taxonomy_term_access' => 1);
+      if (array_intersect_key($this->alterTags, $term_access_tags)) {
+        $this->alterTags += $term_access_tags;
+      }
       $hooks = array('query');
       foreach ($this->alterTags as $tag => $value) {
         $hooks[] = 'query_' . $tag;
@@ -1384,6 +1400,8 @@ class SelectQuery extends Query implements SelectQueryInterface {
   }
 
   public function orderBy($field, $direction = 'ASC') {
+    // Only allow ASC and DESC, default to ASC.
+    $direction = strtoupper($direction) == 'DESC' ? 'DESC' : 'ASC';
     $this->order[$field] = $direction;
     return $this;
   }

includes/database/sqlite/database.inc

@@ -250,7 +250,7 @@ class DatabaseConnection_sqlite extends DatabaseConnection {
     $prefixes[$tablename] = '';
     $this->setPrefix($prefixes);
 
-    $this->query(preg_replace('/^SELECT/i', 'CREATE TEMPORARY TABLE ' . $tablename . ' AS SELECT', $query), $args, $options);
+    $this->query('CREATE TEMPORARY TABLE ' . $tablename . ' AS ' . $query, $args, $options);
     return $tablename;
   }
 
@@ -378,6 +378,14 @@ class DatabaseConnection_sqlite extends DatabaseConnection {
     }
   }
 
+  public function utf8mb4IsActive() {
+    return TRUE;
+  }
+
+  public function utf8mb4IsSupported() {
+    return TRUE;
+  }
+
 }
 
 /**

includes/database/sqlite/install.inc

@@ -14,8 +14,6 @@ class DatabaseTasks_sqlite extends DatabaseTasks {
 
   /**
    * Minimum engine version.
-   *
-   * @todo: consider upping to 3.6.8 in Drupal 8 to get SAVEPOINT support.
    */
   public function minimumVersion() {
     return '3.3.7';

includes/database/sqlite/query.inc

@@ -41,7 +41,8 @@ class InsertQuery_sqlite extends InsertQuery {
     // If we're selecting from a SelectQuery, finish building the query and
     // pass it back, as any remaining options are irrelevant.
     if (!empty($this->fromQuery)) {
-      return $comments . 'INSERT INTO {' . $this->table . '} (' . implode(', ', $this->insertFields) . ') ' . $this->fromQuery;
+      $insert_fields_string = $this->insertFields ? ' (' . implode(', ', $this->insertFields) . ') ' : ' ';
+      return $comments . 'INSERT INTO {' . $this->table . '}' . $insert_fields_string . $this->fromQuery;
     }
 
     return $comments . 'INSERT INTO {' . $this->table . '} (' . implode(', ', $this->insertFields) . ') VALUES (' . implode(', ', $placeholders) . ')';
@@ -98,16 +99,15 @@ class UpdateQuery_sqlite extends UpdateQuery {
 
 /**
  * SQLite specific implementation of DeleteQuery.
- *
- * When the WHERE is omitted from a DELETE statement and the table being deleted
- * has no triggers, SQLite uses an optimization to erase the entire table content
- * without having to visit each row of the table individually.
- *
- * Prior to SQLite 3.6.5, SQLite does not return the actual number of rows deleted
- * by that optimized "truncate" optimization.
  */
 class DeleteQuery_sqlite extends DeleteQuery {
   public function execute() {
+    // When the WHERE is omitted from a DELETE statement and the table being
+    // deleted has no triggers, SQLite uses an optimization to erase the entire
+    // table content without having to visit each row of the table individually.
+    // Prior to SQLite 3.6.5, SQLite does not return the actual number of rows
+    // deleted by that optimized "truncate" optimization. But we want to return
+    // the number of rows affected, so we calculate it directly.
     if (!count($this->condition)) {
       $total_rows = $this->connection->query('SELECT COUNT(*) FROM {' . $this->connection->escapeTable($this->table) . '}')->fetchField();
       parent::execute();

includes/database/sqlite/schema.inc

@@ -232,10 +232,10 @@ class DatabaseSchema_sqlite extends DatabaseSchema {
 
   public function renameTable($table, $new_name) {
     if (!$this->tableExists($table)) {
-      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot rename %table to %table_new: table %table doesn't exist.", array('%table' => $table, '%table_new' => $new_name)));
+      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot rename @table to @table_new: table @table doesn't exist.", array('@table' => $table, '@table_new' => $new_name)));
     }
     if ($this->tableExists($new_name)) {
-      throw new DatabaseSchemaObjectExistsException(t("Cannot rename %table to %table_new: table %table_new already exists.", array('%table' => $table, '%table_new' => $new_name)));
+      throw new DatabaseSchemaObjectExistsException(t("Cannot rename @table to @table_new: table @table_new already exists.", array('@table' => $table, '@table_new' => $new_name)));
     }
 
     $schema = $this->introspectSchema($table);
@@ -244,7 +244,7 @@ class DatabaseSchema_sqlite extends DatabaseSchema {
     // database. So the syntax '...RENAME TO database.table' would fail.
     // So we must determine the full table name here rather than surrounding
     // the table with curly braces incase the db_prefix contains a reference
-    // to a database outside of our existsing database.
+    // to a database outside of our existing database.
     $info = $this->getPrefixInfo($new_name);
     $this->connection->query('ALTER TABLE {' . $table . '} RENAME TO ' . $info['table']);
 
@@ -278,10 +278,10 @@ class DatabaseSchema_sqlite extends DatabaseSchema {
 
   public function addField($table, $field, $specification, $keys_new = array()) {
     if (!$this->tableExists($table)) {
-      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot add field %table.%field: table doesn't exist.", array('%field' => $field, '%table' => $table)));
+      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot add field @table.@field: table doesn't exist.", array('@field' => $field, '@table' => $table)));
     }
     if ($this->fieldExists($table, $field)) {
-      throw new DatabaseSchemaObjectExistsException(t("Cannot add field %table.%field: field already exists.", array('%field' => $field, '%table' => $table)));
+      throw new DatabaseSchemaObjectExistsException(t("Cannot add field @table.@field: field already exists.", array('@field' => $field, '@table' => $table)));
     }
 
     // SQLite doesn't have a full-featured ALTER TABLE statement. It only
@@ -494,10 +494,10 @@ class DatabaseSchema_sqlite extends DatabaseSchema {
 
   public function changeField($table, $field, $field_new, $spec, $keys_new = array()) {
     if (!$this->fieldExists($table, $field)) {
-      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot change the definition of field %table.%name: field doesn't exist.", array('%table' => $table, '%name' => $field)));
+      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot change the definition of field @table.@name: field doesn't exist.", array('@table' => $table, '@name' => $field)));
     }
     if (($field != $field_new) && $this->fieldExists($table, $field_new)) {
-      throw new DatabaseSchemaObjectExistsException(t("Cannot rename field %table.%name to %name_new: target field already exists.", array('%table' => $table, '%name' => $field, '%name_new' => $field_new)));
+      throw new DatabaseSchemaObjectExistsException(t("Cannot rename field @table.@name to @name_new: target field already exists.", array('@table' => $table, '@name' => $field, '@name_new' => $field_new)));
     }
 
     $old_schema = $this->introspectSchema($table);
@@ -559,10 +559,10 @@ class DatabaseSchema_sqlite extends DatabaseSchema {
 
   public function addIndex($table, $name, $fields) {
     if (!$this->tableExists($table)) {
-      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot add index %name to table %table: table doesn't exist.", array('%table' => $table, '%name' => $name)));
+      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot add index @name to table @table: table doesn't exist.", array('@table' => $table, '@name' => $name)));
     }
     if ($this->indexExists($table, $name)) {
-      throw new DatabaseSchemaObjectExistsException(t("Cannot add index %name to table %table: index already exists.", array('%table' => $table, '%name' => $name)));
+      throw new DatabaseSchemaObjectExistsException(t("Cannot add index @name to table @table: index already exists.", array('@table' => $table, '@name' => $name)));
     }
 
     $schema['indexes'][$name] = $fields;
@@ -591,10 +591,10 @@ class DatabaseSchema_sqlite extends DatabaseSchema {
 
   public function addUniqueKey($table, $name, $fields) {
     if (!$this->tableExists($table)) {
-      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot add unique key %name to table %table: table doesn't exist.", array('%table' => $table, '%name' => $name)));
+      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot add unique key @name to table @table: table doesn't exist.", array('@table' => $table, '@name' => $name)));
     }
     if ($this->indexExists($table, $name)) {
-      throw new DatabaseSchemaObjectExistsException(t("Cannot add unique key %name to table %table: unique key already exists.", array('%table' => $table, '%name' => $name)));
+      throw new DatabaseSchemaObjectExistsException(t("Cannot add unique key @name to table @table: unique key already exists.", array('@table' => $table, '@name' => $name)));
     }
 
     $schema['unique keys'][$name] = $fields;
@@ -617,14 +617,14 @@ class DatabaseSchema_sqlite extends DatabaseSchema {
 
   public function addPrimaryKey($table, $fields) {
     if (!$this->tableExists($table)) {
-      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot add primary key to table %table: table doesn't exist.", array('%table' => $table)));
+      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot add primary key to table @table: table doesn't exist.", array('@table' => $table)));
     }
 
     $old_schema = $this->introspectSchema($table);
     $new_schema = $old_schema;
 
     if (!empty($new_schema['primary key'])) {
-      throw new DatabaseSchemaObjectExistsException(t("Cannot add primary key to table %table: primary key already exists.", array('%table' => $table)));
+      throw new DatabaseSchemaObjectExistsException(t("Cannot add primary key to table @table: primary key already exists.", array('@table' => $table)));
     }
 
     $new_schema['primary key'] = $fields;
@@ -646,7 +646,7 @@ class DatabaseSchema_sqlite extends DatabaseSchema {
 
   public function fieldSetDefault($table, $field, $default) {
     if (!$this->fieldExists($table, $field)) {
-      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot set default value of field %table.%field: field doesn't exist.", array('%table' => $table, '%field' => $field)));
+      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot set default value of field @table.@field: field doesn't exist.", array('@table' => $table, '@field' => $field)));
     }
 
     $old_schema = $this->introspectSchema($table);
@@ -658,7 +658,7 @@ class DatabaseSchema_sqlite extends DatabaseSchema {
 
   public function fieldSetNoDefault($table, $field) {
     if (!$this->fieldExists($table, $field)) {
-      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot remove default value of field %table.%field: field doesn't exist.", array('%table' => $table, '%field' => $field)));
+      throw new DatabaseSchemaObjectDoesNotExistException(t("Cannot remove default value of field @table.@field: field doesn't exist.", array('@table' => $table, '@field' => $field)));
     }
 
     $old_schema = $this->introspectSchema($table);

includes/date.inc

@@ -12,11 +12,6 @@ function system_default_date_formats() {
   $formats = array();
 
   // Short date formats.
-  $formats[] = array(
-    'type' => 'short',
-    'format' => 'Y-m-d H:i',
-    'locales' => array(),
-  );
   $formats[] = array(
     'type' => 'short',
     'format' => 'm/d/Y - H:i',
@@ -37,6 +32,11 @@ function system_default_date_formats() {
     'format' => 'd.m.Y - H:i',
     'locales' => array('de-ch', 'de-de', 'de-lu', 'fi-fi', 'fr-ch', 'is-is', 'pl-pl', 'ro-ro', 'ru-ru'),
   );
+  $formats[] = array(
+    'type' => 'short',
+    'format' => 'Y-m-d H:i',
+    'locales' => array(),
+  );
   $formats[] = array(
     'type' => 'short',
     'format' => 'm/d/Y - g:ia',
@@ -84,11 +84,6 @@ function system_default_date_formats() {
   );
 
   // Medium date formats.
-  $formats[] = array(
-    'type' => 'medium',
-    'format' => 'D, Y-m-d H:i',
-    'locales' => array(),
-  );
   $formats[] = array(
     'type' => 'medium',
     'format' => 'D, m/d/Y - H:i',
@@ -104,6 +99,11 @@ function system_default_date_formats() {
     'format' => 'D, Y/m/d - H:i',
     'locales' => array('en-ca', 'fr-ca', 'no-no', 'sv-se'),
   );
+  $formats[] = array(
+    'type' => 'medium',
+    'format' => 'D, Y-m-d H:i',
+    'locales' => array(),
+  );
   $formats[] = array(
     'type' => 'medium',
     'format' => 'F j, Y - H:i',

includes/entity.inc

@@ -13,14 +13,6 @@
  */
 interface DrupalEntityControllerInterface {
 
-  /**
-   * Constructor.
-   *
-   * @param $entityType
-   *   The entity type for which the instance is created.
-   */
-  public function __construct($entityType);
-
   /**
    * Resets the internal, static entity cache.
    *
@@ -36,7 +28,9 @@ interface DrupalEntityControllerInterface {
    * @param $ids
    *   An array of entity IDs, or FALSE to load all entities.
    * @param $conditions
-   *   An array of conditions in the form 'field' => $value.
+   *   An array of conditions. Keys are field names on the entity's base table.
+   *   Values will be compared for equality. All the comparisons will be ANDed
+   *   together. This parameter is deprecated; use an EntityFieldQuery instead.
    *
    * @return
    *   An array of entity objects indexed by their ids. When no results are
@@ -54,7 +48,7 @@ interface DrupalEntityControllerInterface {
 class DrupalDefaultEntityController implements DrupalEntityControllerInterface {
 
   /**
-   * Static cache of entities.
+   * Static cache of entities, keyed by entity ID.
    *
    * @var array
    */
@@ -119,6 +113,9 @@ class DrupalDefaultEntityController implements DrupalEntityControllerInterface {
 
   /**
    * Constructor: sets basic variables.
+   *
+   * @param $entityType
+   *   The entity type for which the instance is created.
    */
   public function __construct($entityType) {
     $this->entityType = $entityType;
@@ -186,6 +183,11 @@ class DrupalDefaultEntityController implements DrupalEntityControllerInterface {
       }
     }
 
+    // Ensure integer entity IDs are valid.
+    if (!empty($ids)) {
+      $this->cleanIds($ids);
+    }
+
     // Load any remaining entities from the database. This is the case if $ids
     // is set to FALSE (so we load all entities), if there are any ids left to
     // load, if loading a revision, or if $conditions was passed without $ids.
@@ -226,6 +228,35 @@ class DrupalDefaultEntityController implements DrupalEntityControllerInterface {
     return $entities;
   }
 
+  /**
+   * Ensures integer entity IDs are valid.
+   *
+   * The identifier sanitization provided by this method has been introduced
+   * as Drupal used to rely on the database to facilitate this, which worked
+   * correctly with MySQL but led to errors with other DBMS such as PostgreSQL.
+   *
+   * @param array $ids
+   *   The entity IDs to verify. Non-integer IDs are removed from this array if
+   *   the entity type requires IDs to be integers.
+   */
+  protected function cleanIds(&$ids) {
+    $entity_info = entity_get_info($this->entityType);
+    if (isset($entity_info['base table field types'])) {
+      $id_type = $entity_info['base table field types'][$this->idKey];
+      if ($id_type == 'serial' || $id_type == 'int') {
+        $ids = array_filter($ids, array($this, 'filterId'));
+        $ids = array_map('intval', $ids);
+      }
+    }
+  }
+
+  /**
+   * Callback for array_filter that removes non-integer IDs.
+   */
+  protected function filterId($id) {
+    return is_numeric($id) && $id == (int) $id;
+  }
+
   /**
    * Builds the query to load the entity.
    *
@@ -241,7 +272,9 @@ class DrupalDefaultEntityController implements DrupalEntityControllerInterface {
    * @param $ids
    *   An array of entity IDs, or FALSE to load all entities.
    * @param $conditions
-   *   An array of conditions in the form 'field' => $value.
+   *   An array of conditions. Keys are field names on the entity's base table.
+   *   Values will be compared for equality. All the comparisons will be ANDed
+   *   together. This parameter is deprecated; use an EntityFieldQuery instead.
    * @param $revision_id
    *   The ID of the revision to load, or FALSE if this query is asking for the
    *   most current revision(s).
@@ -365,9 +398,23 @@ class DrupalDefaultEntityController implements DrupalEntityControllerInterface {
     // This ensures the same behavior whether loading from memory or database.
     if ($conditions) {
       foreach ($entities as $entity) {
-        $entity_values = (array) $entity;
-        if (array_diff_assoc($conditions, $entity_values)) {
-          unset($entities[$entity->{$this->idKey}]);
+        // Iterate over all conditions and compare them to the entity
+        // properties. We cannot use array_diff_assoc() here since the
+        // conditions can be nested arrays, too.
+        foreach ($conditions as $property_name => $condition) {
+          if (is_array($condition)) {
+            // Multiple condition values for one property are treated as OR
+            // operation: only if the value is not at all in the condition array
+            // we remove the entity.
+            if (!in_array($entity->{$property_name}, $condition)) {
+              unset($entities[$entity->{$this->idKey}]);
+              continue 2;
+            }
+          }
+          elseif ($condition != $entity->{$property_name}) {
+            unset($entities[$entity->{$this->idKey}]);
+            continue 2;
+          }
         }
       }
     }
@@ -399,7 +446,7 @@ class EntityFieldQueryException extends Exception {}
  *
  * This class allows finding entities based on entity properties (for example,
  * node->changed), field values, and generic entity meta data (bundle,
- * entity type, entity id, and revision ID). It is not possible to query across
+ * entity type, entity ID, and revision ID). It is not possible to query across
  * multiple entity types. For example, there is no facility to find published
  * nodes written by users created in the last hour, as this would require
  * querying both node->status and user->created.
@@ -634,21 +681,43 @@ class EntityFieldQuery {
 
   /**
    * Adds a condition on field values.
-   * 
+   *
    * Note that entities with empty field values will be excluded from the
    * EntityFieldQuery results when using this method.
    *
    * @param $field
    *   Either a field name or a field array.
    * @param $column
-   *   The column that should hold the value to be matched.
+   *   The column that should hold the value to be matched, defined in the
+   *   hook_field_schema() of this field. If this is omitted then all of the
+   *   other parameters are ignored, except $field, and this call will just be
+   *   adding a condition that says that the field has a value, rather than
+   *   testing the value itself.
    * @param $value
-   *   The value to test the column value against.
+   *   The value to test the column value against. In most cases, this is a
+   *   scalar. For more complex options, it is an array. The meaning of each
+   *   element in the array is dependent on $operator.
    * @param $operator
-   *   The operator to be used to test the given value.
+   *   The operator to be used to test the given value. The possible values are:
+   *   - '=', '<>', '>', '>=', '<', '<=', 'STARTS_WITH', 'CONTAINS': These
+   *     operators expect $value to be a literal of the same type as the
+   *     column.
+   *   - 'IN', 'NOT IN': These operators expect $value to be an array of
+   *     literals of the same type as the column.
+   *   - 'BETWEEN': This operator expects $value to be an array of two literals
+   *     of the same type as the column.
+   *   The operator can be omitted, and will default to 'IN' if the value is an
+   *   array, or to '=' otherwise.
    * @param $delta_group
    *   An arbitrary identifier: conditions in the same group must have the same
-   *   $delta_group.
+   *   $delta_group. For example, let's presume a multivalue field which has
+   *   two columns, 'color' and 'shape', and for entity ID 1, there are two
+   *   values: red/square and blue/circle. Entity ID 1 does not have values
+   *   corresponding to 'red circle'; however if you pass 'red' and 'circle' as
+   *   conditions, it will appear in the results -- by default queries will run
+   *   against any combination of deltas. By passing the conditions with the
+   *   same $delta_group it will ensure that only values attached to the same
+   *   delta are matched, and entity 1 would then be excluded from the results.
    * @param $language_group
    *   An arbitrary identifier: conditions in the same group must have the same
    *   $language_group.
@@ -723,9 +792,11 @@ class EntityFieldQuery {
    * @param $field
    *   Either a field name or a field array.
    * @param $column
-   *   A column defined in the hook_field_schema() of this field. If this is
-   *   omitted then the query will find only entities that have data in this
-   *   field, using the entity and property conditions if there are any.
+   *   The column that should hold the value to be matched, defined in the
+   *   hook_field_schema() of this field. If this is omitted then all of the
+   *   other parameters are ignored, except $field, and this call will just be
+   *   adding a condition that says that the field has a value, rather than
+   *   testing the value itself.
    * @param $value
    *   The value to test the column value against. In most cases, this is a
    *   scalar. For more complex options, it is an array. The meaning of each
@@ -744,10 +815,10 @@ class EntityFieldQuery {
    * @param $delta_group
    *   An arbitrary identifier: conditions in the same group must have the same
    *   $delta_group. For example, let's presume a multivalue field which has
-   *   two columns, 'color' and 'shape', and for entity id 1, there are two
+   *   two columns, 'color' and 'shape', and for entity ID 1, there are two
    *   values: red/square and blue/circle. Entity ID 1 does not have values
    *   corresponding to 'red circle', however if you pass 'red' and 'circle' as
-   *   conditions, it will appear in the  results - by default queries will run
+   *   conditions, it will appear in the results -- by default queries will run
    *   against any combination of deltas. By passing the conditions with the
    *   same $delta_group it will ensure that only values attached to the same
    *   delta are matched, and entity 1 would then be excluded from the results.

includes/errors.inc

@@ -9,7 +9,7 @@
  * Maps PHP error constants to watchdog severity levels.
  *
  * The error constants are documented at
- * http://php.net/manual/en/errorfunc.constants.php
+ * http://php.net/manual/errorfunc.constants.php
  *
  * @ingroup logging_severity_levels
  */
@@ -66,7 +66,7 @@ function _drupal_error_handler_real($error_level, $message, $filename, $line, $c
     _drupal_log_error(array(
       '%type' => isset($types[$error_level]) ? $severity_msg : 'Unknown error',
       // The standard PHP error handler considers that the error messages
-      // are HTML. We mimick this behavior here.
+      // are HTML. We mimic this behavior here.
       '!message' => filter_xss_admin($message),
       '%function' => $caller['function'],
       '%file' => $caller['file'],
@@ -114,7 +114,7 @@ function _drupal_decode_exception($exception) {
   return array(
     '%type' => get_class($exception),
     // The standard PHP exception handler considers that the exception message
-    // is plain-text. We mimick this behavior here.
+    // is plain-text. We mimic this behavior here.
     '!message' => check_plain($message),
     '%function' => $caller['function'],
     '%file' => $caller['file'],
@@ -169,7 +169,7 @@ function error_displayable($error = NULL) {
  *   TRUE if the error is fatal.
  */
 function _drupal_log_error($error, $fatal = FALSE) {
-  // Initialize a maintenance theme if the boostrap was not complete.
+  // Initialize a maintenance theme if the bootstrap was not complete.
   // Do it early because drupal_set_message() triggers a drupal_theme_initialize().
   if ($fatal && (drupal_get_bootstrap_phase() != DRUPAL_BOOTSTRAP_FULL)) {
     unset($GLOBALS['theme']);
@@ -199,7 +199,16 @@ function _drupal_log_error($error, $fatal = FALSE) {
     $number++;
   }
 
-  watchdog('php', '%type: !message in %function (line %line of %file).', $error, $error['severity_level']);
+  // Log the error immediately, unless this is a non-fatal error which has been
+  // triggered via drupal_trigger_error_with_delayed_logging(); in that case
+  // trigger it in a shutdown function. Fatal errors are always triggered
+  // immediately since for a fatal error the page request will end here anyway.
+  if (!$fatal && drupal_static('_drupal_trigger_error_with_delayed_logging')) {
+    drupal_register_shutdown_function('watchdog', 'php', '%type: !message in %function (line %line of %file).', $error, $error['severity_level']);
+  }
+  else {
+    watchdog('php', '%type: !message in %function (line %line of %file).', $error, $error['severity_level']);
+  }
 
   if ($fatal) {
     drupal_add_http_header('Status', '500 Service unavailable (with message)');
@@ -224,7 +233,7 @@ function _drupal_log_error($error, $fatal = FALSE) {
   }
   else {
     // Display the message if the current error reporting level allows this type
-    // of message to be displayed, and unconditionnaly in update.php.
+    // of message to be displayed, and unconditionally in update.php.
     if (error_displayable($error)) {
       $class = 'error';
 

includes/file.inc

@@ -273,7 +273,9 @@ function file_default_scheme() {
  *   The normalized URI.
  */
 function file_stream_wrapper_uri_normalize($uri) {
-  $scheme = file_uri_scheme($uri);
+  // Inline file_uri_scheme() function call for performance reasons.
+  $position = strpos($uri, '://');
+  $scheme = $position ? substr($uri, 0, $position) : FALSE;
 
   if ($scheme && file_stream_wrapper_valid_scheme($scheme)) {
     $target = file_uri_target($uri);
@@ -470,8 +472,11 @@ function file_ensure_htaccess() {
  * @param $private
  *   FALSE indicates that $directory should be an open and public directory.
  *   The default is TRUE which indicates a private and protected directory.
+ * @param $force_overwrite
+ *   Set to TRUE to attempt to overwrite the existing .htaccess file if one is
+ *   already present. Defaults to FALSE.
  */
-function file_create_htaccess($directory, $private = TRUE) {
+function file_create_htaccess($directory, $private = TRUE, $force_overwrite = FALSE) {
   if (file_uri_scheme($directory)) {
     $directory = file_stream_wrapper_uri_normalize($directory);
   }
@@ -480,19 +485,12 @@ function file_create_htaccess($directory, $private = TRUE) {
   }
   $htaccess_path =  $directory . '/.htaccess';
 
-  if (file_exists($htaccess_path)) {
+  if (file_exists($htaccess_path) && !$force_overwrite) {
     // Short circuit if the .htaccess file already exists.
     return;
   }
 
-  if ($private) {
-    // Private .htaccess file.
-    $htaccess_lines = "SetHandler Drupal_Security_Do_Not_Remove_See_SA_2006_006\nDeny from all\nOptions None\nOptions +FollowSymLinks";
-  }
-  else {
-    // Public .htaccess file.
-    $htaccess_lines = "SetHandler Drupal_Security_Do_Not_Remove_See_SA_2006_006\nOptions None\nOptions +FollowSymLinks";
-  }
+  $htaccess_lines = file_htaccess_lines($private);
 
   // Write the .htaccess file.
   if (file_put_contents($htaccess_path, $htaccess_lines)) {
@@ -504,6 +502,59 @@ function file_create_htaccess($directory, $private = TRUE) {
   }
 }
 
+/**
+ * Returns the standard .htaccess lines that Drupal writes to file directories.
+ *
+ * @param $private
+ *   (Optional) Set to FALSE to return the .htaccess lines for an open and
+ *   public directory. The default is TRUE, which returns the .htaccess lines
+ *   for a private and protected directory.
+ *
+ * @return
+ *   A string representing the desired contents of the .htaccess file.
+ *
+ * @see file_create_htaccess()
+ */
+function file_htaccess_lines($private = TRUE) {
+  $lines = <<<EOF
+# Turn off all options we don't need.
+Options None
+Options +FollowSymLinks
+
+# Set the catch-all handler to prevent scripts from being executed.
+SetHandler Drupal_Security_Do_Not_Remove_See_SA_2006_006
+<Files *>
+  # Override the handler again if we're run later in the evaluation list.
+  SetHandler Drupal_Security_Do_Not_Remove_See_SA_2013_003
+</Files>
+
+# If we know how to do it safely, disable the PHP engine entirely.
+<IfModule mod_php5.c>
+  php_flag engine off
+</IfModule>
+<IfModule mod_php7.c>
+  php_flag engine off
+</IfModule>
+EOF;
+
+  if ($private) {
+    $lines = <<<EOF
+# Deny all requests from Apache 2.4+.
+<IfModule mod_authz_core.c>
+  Require all denied
+</IfModule>
+
+# Deny all requests from Apache 2.0-2.2.
+<IfModule !mod_authz_core.c>
+  Deny from all
+</IfModule>
+EOF
+    . "\n\n" . $lines;
+  }
+
+  return $lines;
+}
+
 /**
  * Loads file objects from the database.
  *
@@ -586,7 +637,11 @@ function file_save(stdClass $file) {
     module_invoke_all('entity_update', $file, 'file');
   }
 
+  // Clear internal properties.
   unset($file->original);
+  // Clear the static loading cache.
+  entity_get_controller('file')->resetCache(array($file->fid));
+
   return $file;
 }
 
@@ -719,10 +774,11 @@ function file_usage_delete(stdClass $file, $module, $type = NULL, $id = NULL, $c
  * stored in the database. This is a powerful function that in many ways
  * performs like an advanced version of copy().
  * - Checks if $source and $destination are valid and readable/writable.
- * - Checks that $source is not equal to $destination; if they are an error
- *   is reported.
  * - If file already exists in $destination either the call will error out,
  *   replace the file or rename the file based on the $replace parameter.
+ * - If the $source and $destination are equal, the behavior depends on the
+ *   $replace parameter. FILE_EXISTS_REPLACE will error out. FILE_EXISTS_RENAME
+ *   will rename the file until the $destination is unique.
  * - Adds the new file to the files database. If the source file is a
  *   temporary file, the resulting file will also be a temporary file. See
  *   file_save_upload() for details on temporary files.
@@ -817,10 +873,11 @@ function file_valid_uri($uri) {
  * This is a powerful function that in many ways performs like an advanced
  * version of copy().
  * - Checks if $source and $destination are valid and readable/writable.
- * - Checks that $source is not equal to $destination; if they are an error
- *   is reported.
  * - If file already exists in $destination either the call will error out,
  *   replace the file or rename the file based on the $replace parameter.
+ * - If the $source and $destination are equal, the behavior depends on the
+ *   $replace parameter. FILE_EXISTS_REPLACE will error out. FILE_EXISTS_RENAME
+ *   will rename the file until the $destination is unique.
  * - Provides a fallback using realpaths if the move fails using stream
  *   wrappers. This can occur because PHP's copy() function does not properly
  *   support streams if safe_mode or open_basedir are enabled. See
@@ -846,7 +903,6 @@ function file_valid_uri($uri) {
  */
 function file_unmanaged_copy($source, $destination = NULL, $replace = FILE_EXISTS_RENAME) {
   $original_source = $source;
-  $original_destination = $destination;
 
   // Assert that the source file actually exists.
   if (!file_exists($source)) {
@@ -940,8 +996,15 @@ function file_build_uri($path) {
  * @return
  *   The destination filepath, or FALSE if the file already exists
  *   and FILE_EXISTS_ERROR is specified.
+ *
+ * @throws RuntimeException
+ *   Thrown if the filename contains invalid UTF-8.
  */
 function file_destination($destination, $replace) {
+  $basename = drupal_basename($destination);
+  if (!drupal_validate_utf8($basename)) {
+    throw new RuntimeException(sprintf("Invalid filename '%s'", $basename));
+  }
   if (file_exists($destination)) {
     switch ($replace) {
       case FILE_EXISTS_REPLACE:
@@ -949,7 +1012,6 @@ function file_destination($destination, $replace) {
         break;
 
       case FILE_EXISTS_RENAME:
-        $basename = drupal_basename($destination);
         $directory = drupal_dirname($destination);
         $destination = file_create_filename($basename, $directory);
         break;
@@ -1108,10 +1170,10 @@ function file_munge_filename($filename, $extensions, $alerts = TRUE) {
 
   // Allow potentially insecure uploads for very savvy users and admin
   if (!variable_get('allow_insecure_uploads', 0)) {
-    // Remove any null bytes. See http://php.net/manual/en/security.filesystem.nullbytes.php
+    // Remove any null bytes. See http://php.net/manual/security.filesystem.nullbytes.php
     $filename = str_replace(chr(0), '', $filename);
 
-    $whitelist = array_unique(explode(' ', trim($extensions)));
+    $whitelist = array_unique(explode(' ', strtolower(trim($extensions))));
 
     // Split the filename up by periods. The first part becomes the basename
     // the last part the final extension.
@@ -1124,7 +1186,7 @@ function file_munge_filename($filename, $extensions, $alerts = TRUE) {
     // of allowed extensions.
     foreach ($filename_parts as $filename_part) {
       $new_filename .= '.' . $filename_part;
-      if (!in_array($filename_part, $whitelist) && preg_match("/^[a-zA-Z]{2,5}\d?$/", $filename_part)) {
+      if (!in_array(strtolower($filename_part), $whitelist) && preg_match("/^[a-zA-Z]{2,5}\d?$/", $filename_part)) {
         $new_filename .= '_';
       }
     }
@@ -1165,11 +1227,20 @@ function file_unmunge_filename($filename) {
  * @return
  *   File path consisting of $directory and a unique filename based off
  *   of $basename.
+ *
+ * @throws RuntimeException
+ *   Thrown if the $basename is not valid UTF-8 or another error occurs
+ *   stripping control characters.
  */
 function file_create_filename($basename, $directory) {
+  $original = $basename;
   // Strip control characters (ASCII value < 32). Though these are allowed in
   // some filesystems, not many applications handle them well.
   $basename = preg_replace('/[\x00-\x1F]/u', '_', $basename);
+  if (preg_last_error() !== PREG_NO_ERROR) {
+    throw new RuntimeException(sprintf("Invalid filename '%s'", $original));
+  }
+
   if (substr(PHP_OS, 0, 3) == 'WIN') {
     // These characters are not allowed in Windows filenames
     $basename = str_replace(array(':', '*', '?', '"', '<', '>', '|'), '_', $basename);
@@ -1256,6 +1327,7 @@ function file_delete(stdClass $file, $force = FALSE) {
   if (file_unmanaged_delete($file->uri)) {
     db_delete('file_managed')->condition('fid', $file->fid)->execute();
     db_delete('file_usage')->condition('fid', $file->fid)->execute();
+    entity_get_controller('file')->resetCache();
     return TRUE;
   }
   return FALSE;
@@ -1365,8 +1437,9 @@ function file_space_used($uid = NULL, $status = FILE_STATUS_PERMANENT) {
  * Temporary files are periodically cleaned. To make the file a permanent file,
  * assign the status and use file_save() to save the changes.
  *
- * @param $source
- *   A string specifying the filepath or URI of the uploaded file to save.
+ * @param $form_field_name
+ *   A string that is the associative array key of the upload form element in
+ *   the form array.
  * @param $validators
  *   An optional, associative array of callback functions used to validate the
  *   file. See file_validate() for a full discussion of the array format.
@@ -1377,9 +1450,9 @@ function file_space_used($uid = NULL, $status = FILE_STATUS_PERMANENT) {
  *   (Beware: this is not safe and should only be allowed for trusted users, if
  *   at all).
  * @param $destination
- *   A string containing the URI $source should be copied to.
- *   This must be a stream wrapper URI. If this value is omitted, Drupal's
- *   temporary files scheme will be used ("temporary://").
+ *   A string containing the URI that the file should be copied to. This must
+ *   be a stream wrapper URI. If this value is omitted, Drupal's temporary
+ *   files scheme will be used ("temporary://").
  * @param $replace
  *   Replace behavior when the destination file already exists:
  *   - FILE_EXISTS_REPLACE: Replace the existing file.
@@ -1397,45 +1470,45 @@ function file_space_used($uid = NULL, $status = FILE_STATUS_PERMANENT) {
  *   - source: Path to the file before it is moved.
  *   - destination: Path to the file after it is moved (same as 'uri').
  */
-function file_save_upload($source, $validators = array(), $destination = FALSE, $replace = FILE_EXISTS_RENAME) {
+function file_save_upload($form_field_name, $validators = array(), $destination = FALSE, $replace = FILE_EXISTS_RENAME) {
   global $user;
   static $upload_cache;
 
   // Return cached objects without processing since the file will have
   // already been processed and the paths in _FILES will be invalid.
-  if (isset($upload_cache[$source])) {
-    return $upload_cache[$source];
+  if (isset($upload_cache[$form_field_name])) {
+    return $upload_cache[$form_field_name];
   }
 
   // Make sure there's an upload to process.
-  if (empty($_FILES['files']['name'][$source])) {
+  if (empty($_FILES['files']['name'][$form_field_name])) {
     return NULL;
   }
 
   // Check for file upload errors and return FALSE if a lower level system
   // error occurred. For a complete list of errors:
-  // See http://php.net/manual/en/features.file-upload.errors.php.
-  switch ($_FILES['files']['error'][$source]) {
+  // See http://php.net/manual/features.file-upload.errors.php.
+  switch ($_FILES['files']['error'][$form_field_name]) {
     case UPLOAD_ERR_INI_SIZE:
     case UPLOAD_ERR_FORM_SIZE:
-      drupal_set_message(t('The file %file could not be saved, because it exceeds %maxsize, the maximum allowed size for uploads.', array('%file' => $_FILES['files']['name'][$source], '%maxsize' => format_size(file_upload_max_size()))), 'error');
+      drupal_set_message(t('The file %file could not be saved, because it exceeds %maxsize, the maximum allowed size for uploads.', array('%file' => $_FILES['files']['name'][$form_field_name], '%maxsize' => format_size(file_upload_max_size()))), 'error');
       return FALSE;
 
     case UPLOAD_ERR_PARTIAL:
     case UPLOAD_ERR_NO_FILE:
-      drupal_set_message(t('The file %file could not be saved, because the upload did not complete.', array('%file' => $_FILES['files']['name'][$source])), 'error');
+      drupal_set_message(t('The file %file could not be saved, because the upload did not complete.', array('%file' => $_FILES['files']['name'][$form_field_name])), 'error');
       return FALSE;
 
     case UPLOAD_ERR_OK:
       // Final check that this is a valid upload, if it isn't, use the
       // default error handler.
-      if (is_uploaded_file($_FILES['files']['tmp_name'][$source])) {
+      if (is_uploaded_file($_FILES['files']['tmp_name'][$form_field_name])) {
         break;
       }
 
     // Unknown error
     default:
-      drupal_set_message(t('The file %file could not be saved. An unknown error has occurred.', array('%file' => $_FILES['files']['name'][$source])), 'error');
+      drupal_set_message(t('The file %file could not be saved. An unknown error has occurred.', array('%file' => $_FILES['files']['name'][$form_field_name])), 'error');
       return FALSE;
   }
 
@@ -1443,10 +1516,10 @@ function file_save_upload($source, $validators = array(), $destination = FALSE,
   $file = new stdClass();
   $file->uid      = $user->uid;
   $file->status   = 0;
-  $file->filename = trim(drupal_basename($_FILES['files']['name'][$source]), '.');
-  $file->uri      = $_FILES['files']['tmp_name'][$source];
+  $file->filename = trim(drupal_basename($_FILES['files']['name'][$form_field_name]), '.');
+  $file->uri      = $_FILES['files']['tmp_name'][$form_field_name];
   $file->filemime = file_get_mimetype($file->filename);
-  $file->filesize = $_FILES['files']['size'][$source];
+  $file->filesize = $_FILES['files']['size'][$form_field_name];
 
   $extensions = '';
   if (isset($validators['file_validate_extensions'])) {
@@ -1479,9 +1552,9 @@ function file_save_upload($source, $validators = array(), $destination = FALSE,
   // rename filename.php.foo and filename.php to filename.php.foo.txt and
   // filename.php.txt, respectively). Don't rename if 'allow_insecure_uploads'
   // evaluates to TRUE.
-  if (!variable_get('allow_insecure_uploads', 0) && preg_match('/\.(php|pl|py|cgi|asp|js)(\.|$)/i', $file->filename) && (substr($file->filename, -4) != '.txt')) {
+  if (!variable_get('allow_insecure_uploads', 0) && preg_match('/\.(php|phar|pl|py|cgi|asp|js)(\.|$)/i', $file->filename) && (substr($file->filename, -4) != '.txt')) {
     $file->filemime = 'text/plain';
-    $file->uri .= '.txt';
+    // The destination filename will also later be used to create the URI.
     $file->filename .= '.txt';
     // The .txt extension may not be in the allowed list of extensions. We have
     // to add it here or else the file upload will fail.
@@ -1503,20 +1576,26 @@ function file_save_upload($source, $validators = array(), $destination = FALSE,
     return FALSE;
   }
 
-  $file->source = $source;
+  $file->source = $form_field_name;
   // A URI may already have a trailing slash or look like "public://".
   if (substr($destination, -1) != '/') {
     $destination .= '/';
   }
-  $file->destination = file_destination($destination . $file->filename, $replace);
+  try {
+    $file->destination = file_destination($destination . $file->filename, $replace);
+  }
+  catch (RuntimeException $e) {
+    drupal_set_message(t('The file %source could not be uploaded because the name is invalid.', array('%source' => $form_field_name)), 'error');
+    return FALSE;
+  }
   // If file_destination() returns FALSE then $replace == FILE_EXISTS_ERROR and
   // there's an existing file so we need to bail.
   if ($file->destination === FALSE) {
-    drupal_set_message(t('The file %source could not be uploaded because a file by that name already exists in the destination %directory.', array('%source' => $source, '%directory' => $destination)), 'error');
+    drupal_set_message(t('The file %source could not be uploaded because a file by that name already exists in the destination %directory.', array('%source' => $form_field_name, '%directory' => $destination)), 'error');
     return FALSE;
   }
 
-  // Add in our check of the the file name length.
+  // Add in our check of the file name length.
   $validators['file_validate_name_length'] = array();
 
   // Call the validation functions specified by this function's caller.
@@ -1531,7 +1610,7 @@ function file_save_upload($source, $validators = array(), $destination = FALSE,
     else {
       $message .= ' ' . array_pop($errors);
     }
-    form_set_error($source, $message);
+    form_set_error($form_field_name, $message);
     return FALSE;
   }
 
@@ -1539,8 +1618,8 @@ function file_save_upload($source, $validators = array(), $destination = FALSE,
   // directory. This overcomes open_basedir restrictions for future file
   // operations.
   $file->uri = $file->destination;
-  if (!drupal_move_uploaded_file($_FILES['files']['tmp_name'][$source], $file->uri)) {
-    form_set_error($source, t('File upload error. Could not move uploaded file.'));
+  if (!drupal_move_uploaded_file($_FILES['files']['tmp_name'][$form_field_name], $file->uri)) {
+    form_set_error($form_field_name, t('File upload error. Could not move uploaded file.'));
     watchdog('file', 'Upload error. Could not move uploaded file %file to destination %destination.', array('%file' => $file->filename, '%destination' => $file->uri));
     return FALSE;
   }
@@ -1559,8 +1638,22 @@ function file_save_upload($source, $validators = array(), $destination = FALSE,
 
   // If we made it this far it's safe to record this file in the database.
   if ($file = file_save($file)) {
+    // Track non-public files in the session if they were uploaded by an
+    // anonymous user. This allows modules such as the File module to only
+    // grant view access to the specific anonymous user who uploaded the file.
+    // See file_file_download().
+    // The 'file_public_schema' variable is used to allow other publicly
+    // accessible file schemes to be treated the same as the public:// scheme
+    // provided by Drupal core and to avoid adding unnecessary data to the
+    // session (and the resulting bypass of the page cache) in those cases. For
+    // security reasons, only schemes that are completely publicly accessible,
+    // with no download restrictions, should be added to this variable. See
+    // file_managed_file_value().
+    if (!$user->uid && !in_array($destination_scheme, variable_get('file_public_schema', array('public')))) {
+      $_SESSION['anonymous_allowed_file_ids'][$file->fid] = $file->fid;
+    }
     // Add file to the cache.
-    $upload_cache[$source] = $file;
+    $upload_cache[$form_field_name] = $file;
     return $file;
   }
   return FALSE;
@@ -1686,8 +1779,6 @@ function file_validate_extensions(stdClass $file, $extensions) {
 /**
  * Checks that the file's size is below certain limits.
  *
- * This check is not enforced for the user #1.
- *
  * @param $file
  *   A Drupal file object.
  * @param $file_limit
@@ -1705,20 +1796,17 @@ function file_validate_extensions(stdClass $file, $extensions) {
  */
 function file_validate_size(stdClass $file, $file_limit = 0, $user_limit = 0) {
   global $user;
-
   $errors = array();
 
-  // Bypass validation for uid  = 1.
-  if ($user->uid != 1) {
-    if ($file_limit && $file->filesize > $file_limit) {
-      $errors[] = t('The file is %filesize exceeding the maximum file size of %maxsize.', array('%filesize' => format_size($file->filesize), '%maxsize' => format_size($file_limit)));
-    }
+  if ($file_limit && $file->filesize > $file_limit) {
+    $errors[] = t('The file is %filesize exceeding the maximum file size of %maxsize.', array('%filesize' => format_size($file->filesize), '%maxsize' => format_size($file_limit)));
+  }
 
-    // Save a query by only calling file_space_used() when a limit is provided.
-    if ($user_limit && (file_space_used($user->uid) + $file->filesize) > $user_limit) {
-      $errors[] = t('The file is %filesize which would exceed your disk quota of %quota.', array('%filesize' => format_size($file->filesize), '%quota' => format_size($user_limit)));
-    }
+  // Save a query by only calling file_space_used() when a limit is provided.
+  if ($user_limit && (file_space_used($user->uid) + $file->filesize) > $user_limit) {
+    $errors[] = t('The file is %filesize which would exceed your disk quota of %quota.', array('%filesize' => format_size($file->filesize), '%quota' => format_size($user_limit)));
   }
+
   return $errors;
 }
 
@@ -1747,7 +1835,7 @@ function file_validate_is_image(stdClass $file) {
 /**
  * Verifies that image dimensions are within the specified maximum and minimum.
  *
- * Non-image files will be ignored. If a image toolkit is available the image
+ * Non-image files will be ignored. If an image toolkit is available the image
  * will be scaled to fit within the desired maximum dimensions.
  *
  * @param $file
@@ -1956,23 +2044,7 @@ function file_download() {
   $target = implode('/', $args);
   $uri = $scheme . '://' . $target;
   if (file_stream_wrapper_valid_scheme($scheme) && file_exists($uri)) {
-    // Let other modules provide headers and controls access to the file.
-    // module_invoke_all() uses array_merge_recursive() which merges header
-    // values into a new array. To avoid that and allow modules to override
-    // headers instead, use array_merge() to merge the returned arrays.
-    $headers = array();
-    foreach (module_implements('file_download') as $module) {
-      $function = $module . '_file_download';
-      $result = $function($uri);
-      if ($result == -1) {
-        // Throw away the headers received so far.
-        $headers = array();
-        break;
-      }
-      if (isset($result) && is_array($result)) {
-        $headers = array_merge($headers, $result);
-      }
-    }
+    $headers = file_download_headers($uri);
     if (count($headers)) {
       file_transfer($uri, $headers);
     }
@@ -1984,6 +2056,69 @@ function file_download() {
   drupal_exit();
 }
 
+/**
+ * Retrieves headers for a private file download.
+ *
+ * Calls all module implementations of hook_file_download() to retrieve headers
+ * for files by the module that originally provided the file. The presence of
+ * returned headers indicates the current user has access to the file.
+ *
+ * @param $uri
+ *   The URI for the file whose headers should be retrieved.
+ *
+ * @return
+ *   If access is allowed, headers for the file, suitable for passing to
+ *   file_transfer(). If access is not allowed, an empty array will be returned.
+ *
+ * @see file_transfer()
+ * @see file_download_access()
+ * @see hook_file_download()
+ */
+function file_download_headers($uri) {
+  // Let other modules provide headers and control access to the file.
+  // module_invoke_all() uses array_merge_recursive() which merges header
+  // values into a new array. To avoid that and allow modules to override
+  // headers instead, use array_merge() to merge the returned arrays.
+  $headers = array();
+  foreach (module_implements('file_download') as $module) {
+    $function = $module . '_file_download';
+    $result = $function($uri);
+    if ($result == -1) {
+      // Throw away the headers received so far.
+      $headers = array();
+      break;
+    }
+    if (isset($result) && is_array($result)) {
+      $headers = array_merge($headers, $result);
+    }
+  }
+  return $headers;
+}
+
+/**
+ * Checks that the current user has access to a particular file.
+ *
+ * The return value of this function hinges on the return value from
+ * file_download_headers(), which is the function responsible for collecting
+ * access information through hook_file_download().
+ *
+ * If immediately transferring the file to the browser and the headers will
+ * need to be retrieved, the return value of file_download_headers() should be
+ * used to determine access directly, so that access checks will not be run
+ * twice.
+ *
+ * @param $uri
+ *   The URI for the file whose access should be retrieved.
+ *
+ * @return
+ *   Boolean TRUE if access is allowed. FALSE if access is not allowed.
+ *
+ * @see file_download_headers()
+ * @see hook_file_download()
+ */
+function file_download_access($uri) {
+  return count(file_download_headers($uri)) > 0;
+}
 
 /**
  * Finds all files that match a given mask in a given directory.
@@ -2019,9 +2154,33 @@ function file_download() {
  *   'filename', and 'name' members corresponding to the matching files.
  */
 function file_scan_directory($dir, $mask, $options = array(), $depth = 0) {
+  // Default nomask option.
+  $nomask = '/(\.\.?|CVS)$/';
+
+  // Overrides the $nomask variable accordingly if $options['nomask'] is set.
+  //
+  // Allow directories specified in settings.php to be ignored. You can use this
+  // to not check for files in common special-purpose directories. For example,
+  // node_modules and bower_components. Ignoring irrelevant directories is a
+  // performance boost.
+  if (!isset($options['nomask'])) {
+    $ignore_directories = variable_get(
+      'file_scan_ignore_directories',
+      array()
+    );
+
+    foreach ($ignore_directories as $index => $ignore_directory) {
+      $ignore_directories[$index] = preg_quote($ignore_directory, '/');
+    }
+
+    if (!empty($ignore_directories)) {
+      $nomask = '/^(\.\.?)|CVS|' . implode('|', $ignore_directories) . '$/';
+    }
+  }
+
   // Merge in defaults.
   $options += array(
-    'nomask' => '/(\.\.?|CVS)$/',
+    'nomask' => $nomask,
     'callback' => 0,
     'recurse' => TRUE,
     'key' => 'uri',
@@ -2177,7 +2336,7 @@ function drupal_chmod($uri, $mode = NULL) {
  * @param $uri
  *   A URI or pathname.
  * @param $context
- *   Refer to http://php.net/manual/en/ref.stream.php
+ *   Refer to http://php.net/manual/ref.stream.php
  *
  * @return
  *   Boolean TRUE on success, or FALSE on failure.
@@ -2199,29 +2358,21 @@ function drupal_unlink($uri, $context = NULL) {
 }
 
 /**
- * Returns the absolute local filesystem path of a stream URI.
+ * Resolves the absolute filepath of a local URI or filepath.
  *
- * This function was originally written to ease the conversion of 6.x code to
- * use 7.x stream wrappers. However, it assumes that every URI may be resolved
- * to an absolute local filesystem path, and this assumption fails when stream
- * wrappers are used to support remote file storage. Remote stream wrappers
- * may implement the realpath method by always returning FALSE. The use of
- * drupal_realpath() is discouraged, and is slowly being removed from core
- * functions where possible.
+ * The use of drupal_realpath() is discouraged, because it does not work for
+ * remote URIs. Except in rare cases, URIs should not be manually resolved.
  *
  * Only use this function if you know that the stream wrapper in the URI uses
  * the local file system, and you need to pass an absolute path to a function
  * that is incompatible with stream URIs.
  *
- * @param $uri
- *   A stream wrapper URI or a filesystem path, possibly including one or more
- *   symbolic links.
- *
- * @return
- *   The absolute local filesystem path (with no symbolic links), or FALSE on
- *   failure.
+ * @param string $uri
+ *   A stream wrapper URI or a filepath, possibly including one or more symbolic
+ *   links.
  *
- * @todo This function is deprecated, and should be removed wherever possible.
+ * @return string|false
+ *   The absolute local filepath (with no symbolic links), or FALSE on failure.
  *
  * @see DrupalStreamWrapperInterface::realpath()
  * @see http://php.net/manual/function.realpath.php
@@ -2318,7 +2469,7 @@ function drupal_basename($uri, $suffix = NULL) {
  * @param $recursive
  *   Default to FALSE.
  * @param $context
- *   Refer to http://php.net/manual/en/ref.stream.php
+ *   Refer to http://php.net/manual/ref.stream.php
  *
  * @return
  *   Boolean TRUE on success, or FALSE on failure.
@@ -2349,7 +2500,7 @@ function drupal_mkdir($uri, $mode = NULL, $recursive = FALSE, $context = NULL) {
  * @param $uri
  *   A URI or pathname.
  * @param $context
- *   Refer to http://php.net/manual/en/ref.stream.php
+ *   Refer to http://php.net/manual/ref.stream.php
  *
  * @return
  *   Boolean TRUE on success, or FALSE on failure.
@@ -2474,7 +2625,6 @@ function file_directory_temp() {
  *   An associative array of headers, as expected by file_transfer().
  */
 function file_get_content_headers($file) {
-  $name = mime_header_encode($file->filename);
   $type = mime_header_encode($file->filemime);
 
   return array(

includes/file.mimetypes.inc

@@ -43,6 +43,7 @@ function file_default_mimetype_mapping() {
       4 => 'application/cap',
       5 => 'application/cu-seeme',
       6 => 'application/dsptype',
+      350 => 'application/epub+zip',
       7 => 'application/hta',
       8 => 'application/java-archive',
       9 => 'application/java-serialized-object',
@@ -64,6 +65,7 @@ function file_default_mimetype_mapping() {
       25 => 'application/rss+xml',
       26 => 'application/rtf',
       27 => 'application/smil',
+      349 => 'application/vnd.amazon.ebook',
       28 => 'application/vnd.cinderella',
       29 => 'application/vnd.google-earth.kml+xml',
       30 => 'application/vnd.google-earth.kmz',
@@ -183,6 +185,8 @@ function file_default_mimetype_mapping() {
       144 => 'application/x-lzx',
       145 => 'application/x-maker',
       146 => 'application/x-mif',
+      351 => 'application/x-mobipocket-ebook',
+      352 => 'application/x-mobipocket-ebook',
       147 => 'application/x-ms-wmd',
       148 => 'application/x-ms-wmz',
       149 => 'application/x-msdos-program',
@@ -228,8 +232,10 @@ function file_default_mimetype_mapping() {
       188 => 'audio/mpeg',
       189 => 'audio/ogg',
       190 => 'audio/prs.sid',
+      356 => 'audio/webm',
       191 => 'audio/x-aiff',
       192 => 'audio/x-gsm',
+      354 => 'audio/x-matroska',
       193 => 'audio/x-mpegurl',
       194 => 'audio/x-ms-wax',
       195 => 'audio/x-ms-wma',
@@ -301,6 +307,7 @@ function file_default_mimetype_mapping() {
       261 => 'image/vnd.djvu',
       262 => 'image/vnd.microsoft.icon',
       263 => 'image/vnd.wap.wbmp',
+      355 => 'image/webp',
       264 => 'image/x-cmu-raster',
       265 => 'image/x-coreldraw',
       266 => 'image/x-coreldrawpattern',
@@ -337,6 +344,7 @@ function file_default_mimetype_mapping() {
       297 => 'text/vnd.sun.j2me.app-descriptor',
       298 => 'text/vnd.wap.wml',
       299 => 'text/vnd.wap.wmlscript',
+      358 => 'text/vtt',
       300 => 'text/x-bibtex',
       301 => 'text/x-boo',
       302 => 'text/x-c++hdr',
@@ -371,9 +379,11 @@ function file_default_mimetype_mapping() {
       331 => 'video/ogg',
       332 => 'video/quicktime',
       333 => 'video/vnd.mpegurl',
+      357 => 'video/webm',
       347 => 'video/x-flv',
       334 => 'video/x-la-asf',
       348 => 'video/x-m4v',
+      353 => 'video/x-matroska',
       335 => 'video/x-mng',
       336 => 'video/x-ms-asf',
       337 => 'video/x-ms-wm',
@@ -854,6 +864,16 @@ function file_default_mimetype_mapping() {
       'f4b' => 346,
       'flv' => 347,
       'm4v' => 348,
+      'azw' => 349,
+      'epub' => 350,
+      'mobi' => 351,
+      'prc' => 352,
+      'mkv' => 353,
+      'mka' => 354,
+      'webp' => 355,
+      'weba' => 356,
+      'webm' => 357,
+      'vtt' => 358,
     ),
   );
 }

includes/file.phar.inc

@@ -0,0 +1,55 @@
+<?php
+
+use Drupal\Core\Security\PharExtensionInterceptor;
+use TYPO3\PharStreamWrapper\Manager as PharStreamWrapperManager;
+use TYPO3\PharStreamWrapper\Behavior as PharStreamWrapperBehavior;
+use TYPO3\PharStreamWrapper\PharStreamWrapper;
+
+/**
+ * Registers a phar stream wrapper that is more secure than PHP's built-in one.
+ *
+ * @see file_get_stream_wrappers()
+ */
+function file_register_phar_wrapper() {
+  $directory = DRUPAL_ROOT . '/misc/typo3/phar-stream-wrapper/src';
+  include_once $directory . '/Assertable.php';
+  include_once $directory . '/Behavior.php';
+  include_once $directory . '/Exception.php';
+  include_once $directory . '/Helper.php';
+  include_once $directory . '/Manager.php';
+  include_once $directory . '/PharStreamWrapper.php';
+  include_once $directory . '/Collectable.php';
+  include_once $directory . '/Interceptor/ConjunctionInterceptor.php';
+  include_once $directory . '/Interceptor/PharMetaDataInterceptor.php';
+  include_once $directory . '/Phar/Container.php';
+  include_once $directory . '/Phar/DeserializationException.php';
+  include_once $directory . '/Phar/Manifest.php';
+  include_once $directory . '/Phar/Reader.php';
+  include_once $directory . '/Phar/ReaderException.php';
+  include_once $directory . '/Phar/Stub.php';
+  include_once $directory . '/Resolvable.php';
+  include_once $directory . '/Resolver/PharInvocation.php';
+  include_once $directory . '/Resolver/PharInvocationCollection.php';
+  include_once $directory . '/Resolver/PharInvocationResolver.php';
+  include_once DRUPAL_ROOT . '/misc/typo3/drupal-security/PharExtensionInterceptor.php';
+  include_once DRUPAL_ROOT . '/misc/brumann/polyfill-unserialize/src/Unserialize.php';
+
+  // Set up a stream wrapper to handle insecurities due to PHP's built-in
+  // phar stream wrapper.
+  try {
+    $behavior = new PharStreamWrapperBehavior();
+    PharStreamWrapperManager::initialize(
+      $behavior->withAssertion(new PharExtensionInterceptor())
+    );
+  }
+  catch (\LogicException $e) {
+    // Continue if the PharStreamWrapperManager is already initialized.
+    // For example, this occurs following a drupal_static_reset(), such
+    // as during tests.
+  };
+
+  // To prevent file_stream_wrapper_valid_scheme() treating "phar" as a valid
+  // scheme, this is registered with PHP only, not with  hook_stream_wrappers()
+  // or the internal storage of file_get_stream_wrappers().
+  stream_wrapper_register('phar', '\\TYPO3\\PharStreamWrapper\\PharStreamWrapper');
+}

includes/filetransfer/filetransfer.inc

@@ -301,7 +301,7 @@ abstract class FileTransfer {
     $parts = explode('/', $path);
     $chroot = '';
     while (count($parts)) {
-      $check = implode($parts, '/');
+      $check = implode('/', $parts);
       if ($this->isFile($check . '/' . drupal_basename(__FILE__))) {
         // Remove the trailing slash.
         return substr($chroot, 0, -1);

includes/filetransfer/ftp.inc

@@ -82,11 +82,11 @@ class FileTransferFTPExtension extends FileTransferFTP implements FileTransferCh
     if (!$list) {
       $list = array();
     }
-    foreach ($list as $item){
+    foreach ($list as $item) {
       if ($item == '.' || $item == '..') {
         continue;
       }
-      if (@ftp_chdir($this->connection, $item)){
+      if (@ftp_chdir($this->connection, $item)) {
         ftp_cdup($this->connection);
         $this->removeDirectory(ftp_pwd($this->connection) . '/' . $item);
       }
@@ -122,7 +122,7 @@ class FileTransferFTPExtension extends FileTransferFTP implements FileTransferCh
 
   function chmodJailed($path, $mode, $recursive) {
     if (!ftp_chmod($this->connection, $mode, $path)) {
-      throw new FileTransferException("Unable to set permissions on %file", NULL, array ('%file' => $path));
+      throw new FileTransferException("Unable to set permissions on %file", NULL, array('%file' => $path));
     }
     if ($this->isDirectory($path) && $recursive) {
       $filelist = @ftp_nlist($this->connection, $path);

includes/filetransfer/ssh.inc

@@ -72,7 +72,8 @@ class FileTransferSSH extends FileTransfer implements FileTransferChmodInterface
         return TRUE;
       }
       return FALSE;
-    } else {
+    }
+    else {
       throw new FileTransferException('Cannot check @path.', NULL, array('@path' => $path));
     }
   }
@@ -85,7 +86,8 @@ class FileTransferSSH extends FileTransfer implements FileTransferChmodInterface
         return TRUE;
       }
       return FALSE;
-    } else {
+    }
+    else {
       throw new FileTransferException('Cannot check @path.', NULL, array('@path' => $path));
     }
   }

includes/form.inc

@@ -15,10 +15,9 @@
  * reference the form builder function using \@see. For examples, of this see
  * system_modules_uninstall() or user_pass(), the latter of which has the
  * following in its doxygen documentation:
- *
- * \@ingroup forms
- * \@see user_pass_validate().
- * \@see user_pass_submit().
+ * - \@ingroup forms
+ * - \@see user_pass_validate()
+ * - \@see user_pass_submit()
  *
  * @}
  */
@@ -106,7 +105,8 @@
  *   generate the same form (or very similar forms) using different $form_ids
  *   can implement hook_forms(), which maps different $form_id values to the
  *   proper form constructor function. Examples may be found in node_forms(),
- *   and search_forms().
+ *   and search_forms(). hook_forms() can also be used to define forms in
+ *   classes.
  * @param ...
  *   Any additional arguments are passed on to the functions called by
  *   drupal_get_form(), including the unique form constructor function. For
@@ -168,6 +168,12 @@ function drupal_get_form($form_id) {
  *       processed.
  *     - base_form_id: Identification for a base form, as declared in a
  *       hook_forms() implementation.
+ *     - immutable: If this flag is set to TRUE, a new form build id is
+ *       generated when the form is loaded from the cache. If it is subsequently
+ *       saved to the cache again, it will have another cache id and therefore
+ *       the original form and form-state will remain unaltered. This is
+ *       important when page caching is enabled in order to prevent form state
+ *       from leaking between anonymous users.
  *   - rebuild_info: Internal. Similar to 'build_info', but pertaining to
  *     drupal_rebuild_form().
  *   - rebuild: Normally, after the entire form processing is completed and
@@ -235,6 +241,12 @@ function drupal_get_form($form_id) {
  *     likely to occur during Ajax operations.
  *   - programmed: If TRUE, the form was submitted programmatically, usually
  *     invoked via drupal_form_submit(). Defaults to FALSE.
+ *   - programmed_bypass_access_check: If TRUE, programmatic form submissions
+ *     are processed without taking #access into account. Set this to FALSE
+ *     when submitting a form programmatically with values that may have been
+ *     input by the user executing the current request; this will cause #access
+ *     to be respected as it would on a normal form submission. Defaults to
+ *     TRUE.
  *   - process_input: Boolean flag. TRUE signifies correct form submission.
  *     This is always TRUE for programmed forms coming from drupal_form_submit()
  *     (see 'programmed' key), or if the form_id coming from the $_POST data is
@@ -402,6 +414,7 @@ function form_state_defaults() {
     'submitted' => FALSE,
     'executed' => FALSE,
     'programmed' => FALSE,
+    'programmed_bypass_access_check' => TRUE,
     'cache'=> FALSE,
     'method' => 'post',
     'groups' => array(),
@@ -452,17 +465,25 @@ function drupal_rebuild_form($form_id, &$form_state, $old_form = NULL) {
   $form = drupal_retrieve_form($form_id, $form_state);
 
   // If only parts of the form will be returned to the browser (e.g., Ajax or
-  // RIA clients), re-use the old #build_id to not require client-side code to
-  // manually update the hidden 'build_id' input element.
+  // RIA clients), or if the form already had a new build ID regenerated when it
+  // was retrieved from the form cache, reuse the existing #build_id.
   // Otherwise, a new #build_id is generated, to not clobber the previous
   // build's data in the form cache; also allowing the user to go back to an
   // earlier build, make changes, and re-submit.
   // @see drupal_prepare_form()
-  if (isset($old_form['#build_id']) && !empty($form_state['rebuild_info']['copy']['#build_id'])) {
+  $enforce_old_build_id = isset($old_form['#build_id']) && !empty($form_state['rebuild_info']['copy']['#build_id']);
+  $old_form_is_mutable_copy = isset($old_form['#build_id_old']);
+  if ($enforce_old_build_id || $old_form_is_mutable_copy) {
     $form['#build_id'] = $old_form['#build_id'];
+    if ($old_form_is_mutable_copy) {
+      $form['#build_id_old'] = $old_form['#build_id_old'];
+    }
   }
   else {
-    $form['#build_id'] = 'form-' . drupal_hash_base64(uniqid(mt_rand(), TRUE) . mt_rand());
+    if (isset($old_form['#build_id'])) {
+      $form['#build_id_old'] = $old_form['#build_id'];
+    }
+    $form['#build_id'] = 'form-' . drupal_random_key();
   }
 
   // #action defaults to request_uri(), but in case of Ajax and other partial
@@ -516,6 +537,15 @@ function form_get_cache($form_build_id, &$form_state) {
           }
         }
       }
+      // Generate a new #build_id if the cached form was rendered on a cacheable
+      // page.
+      if (!empty($form_state['build_info']['immutable'])) {
+        $form['#build_id_old'] = $form['#build_id'];
+        $form['#build_id'] = 'form-' . drupal_random_key();
+        $form['form_build_id']['#value'] = $form['#build_id'];
+        $form['form_build_id']['#id'] = $form['#build_id'];
+        unset($form_state['build_info']['immutable']);
+      }
       return $form;
     }
   }
@@ -525,18 +555,33 @@ function form_get_cache($form_build_id, &$form_state) {
  * Stores a form in the cache.
  */
 function form_set_cache($form_build_id, $form, $form_state) {
-  // 6 hours cache life time for forms should be plenty.
-  $expire = 21600;
+  // The default cache_form expiration is 6 hours. On busy sites, the cache_form
+  // table can become very large. A shorter cache lifetime can help to keep the
+  // table's size under control.
+  $expire = variable_get('form_cache_expiration', 21600);
+
+  // Ensure that the form build_id embedded in the form structure is the same as
+  // the one passed in as a parameter. This is an additional safety measure to
+  // prevent legacy code operating directly with form_get_cache and
+  // form_set_cache from accidentally overwriting immutable form state.
+  if ($form['#build_id'] != $form_build_id) {
+    watchdog('form', 'Form build-id mismatch detected while attempting to store a form in the cache.', array(), WATCHDOG_ERROR);
+    return;
+  }
 
   // Cache form structure.
   if (isset($form)) {
     if ($GLOBALS['user']->uid) {
       $form['#cache_token'] = drupal_get_token();
     }
+    unset($form['#build_id_old']);
     cache_set('form_' . $form_build_id, $form, 'cache_form', REQUEST_TIME + $expire);
   }
 
   // Cache form state.
+  if (variable_get('cache', 0) && drupal_page_is_cacheable()) {
+    $form_state['build_info']['immutable'] = TRUE;
+  }
   if ($data = array_diff_key($form_state, array_flip(form_state_keys_no_cache()))) {
     cache_set('form_state_' . $form_build_id, $data, 'cache_form', REQUEST_TIME + $expire);
   }
@@ -727,8 +772,9 @@ function drupal_retrieve_form($form_id, &$form_state) {
   // Record the filepath of the include file containing the original form, so
   // the form builder callbacks can be loaded when the form is being rebuilt
   // from cache on a different path (such as 'system/ajax'). See
-  // form_get_cache().
-  // $menu_get_item() is not available during installation.
+  // form_get_cache(). Don't do this in maintenance mode as Drupal may not be
+  // fully bootstrapped (i.e. during installation) in which case
+  // menu_get_item() is not available.
   if (!isset($form_state['build_info']['files']['menu']) && !defined('MAINTENANCE_MODE')) {
     $item = menu_get_item();
     if (!empty($item['include_file'])) {
@@ -766,7 +812,7 @@ function drupal_retrieve_form($form_id, &$form_state) {
     }
     if (isset($form_definition['callback'])) {
       $callback = $form_definition['callback'];
-      $form_state['build_info']['base_form_id'] = $callback;
+      $form_state['build_info']['base_form_id'] = isset($form_definition['base_form_id']) ? $form_definition['base_form_id'] : $callback;
     }
     // In case $form_state['wrapper_callback'] is not defined already, we also
     // allow hook_forms() to define one.
@@ -787,7 +833,7 @@ function drupal_retrieve_form($form_id, &$form_state) {
   // the actual form builder function ($callback) expects. This allows for
   // pre-populating a form with common elements for certain forms, such as
   // back/next/save buttons in multi-step form wizards. See drupal_build_form().
-  if (isset($form_state['wrapper_callback']) && function_exists($form_state['wrapper_callback'])) {
+  if (isset($form_state['wrapper_callback']) && is_callable($form_state['wrapper_callback'])) {
     $form = call_user_func_array($form_state['wrapper_callback'], $args);
     // Put the prepopulated $form into $args.
     $args[0] = $form;
@@ -895,7 +941,7 @@ function drupal_process_form($form_id, &$form, &$form_state) {
         // after the batch is processed.
       }
 
-      // Set a flag to indicate the the form has been processed and executed.
+      // Set a flag to indicate that the form has been processed and executed.
       $form_state['executed'] = TRUE;
 
       // Redirect the form based on values in $form_state.
@@ -976,7 +1022,7 @@ function drupal_prepare_form($form_id, &$form, &$form_state) {
   // @see drupal_build_form()
   // @see drupal_rebuild_form()
   if (!isset($form['#build_id'])) {
-    $form['#build_id'] = 'form-' . drupal_hash_base64(uniqid(mt_rand(), TRUE) . mt_rand());
+    $form['#build_id'] = 'form-' . drupal_random_key();
   }
   $form['form_build_id'] = array(
     '#type' => 'hidden',
@@ -1085,6 +1131,13 @@ function drupal_prepare_form($form_id, &$form, &$form_state) {
   drupal_alter($hooks, $form, $form_state, $form_id);
 }
 
+/**
+ * Helper function to call form_set_error() if there is a token error.
+ */
+function _drupal_invalid_token_set_form_error() {
+  // Setting this error will cause the form to fail validation.
+  form_set_error('form_token', t('The form has become outdated. Press the back button, copy any unsaved work in the form, and then reload the page.'));
+}
 
 /**
  * Validates user-submitted form data in the $form_state array.
@@ -1119,15 +1172,21 @@ function drupal_validate_form($form_id, &$form, &$form_state) {
   }
 
   // If the session token was set by drupal_prepare_form(), ensure that it
-  // matches the current user's session.
-  if (isset($form['#token'])) {
-    if (!drupal_valid_token($form_state['values']['form_token'], $form['#token'])) {
-      $path = current_path();
-      $query = drupal_get_query_parameters();
-      $url = url($path, array('query' => $query));
-
-      // Setting this error will cause the form to fail validation.
-      form_set_error('form_token', t('The form has become outdated. Copy any unsaved work in the form below and then <a href="@link">reload this page</a>.', array('@link' => $url)));
+  // matches the current user's session. This is duplicate to code in
+  // form_builder() but left to protect any custom form handling code.
+  if (!empty($form['#token'])) {
+    if (!drupal_valid_token($form_state['values']['form_token'], $form['#token']) || !empty($form_state['invalid_token'])) {
+      _drupal_invalid_token_set_form_error();
+      // Ignore all submitted values.
+      $form_state['input'] = array();
+      $_POST = array();
+      // Make sure file uploads do not get processed.
+      $_FILES = array();
+      // Stop here and don't run any further validation handlers, because they
+      // could invoke non-safe operations which opens the door for CSRF
+      // vulnerabilities.
+      $validated_forms[$form_id] = TRUE;
+      return;
     }
   }
 
@@ -1382,10 +1441,12 @@ function _form_validate(&$elements, &$form_state, $form_id = NULL) {
       // length if it's a string, and the item count if it's an array.
       // An unchecked checkbox has a #value of integer 0, different than string
       // '0', which could be a valid value.
-      $is_empty_multiple = (!count($elements['#value']));
+      $is_countable = is_array($elements['#value']) || $elements['#value'] instanceof Countable;
+      $is_empty_multiple = $is_countable && count($elements['#value']) == 0;
       $is_empty_string = (is_string($elements['#value']) && drupal_strlen(trim($elements['#value'])) == 0);
       $is_empty_value = ($elements['#value'] === 0);
-      if ($is_empty_multiple || $is_empty_string || $is_empty_value) {
+      $is_empty_null = is_null($elements['#value']);
+      if ($is_empty_multiple || $is_empty_string || $is_empty_value || $is_empty_null) {
         // Although discouraged, a #title is not mandatory for form elements. In
         // case there is no #title, we cannot set a form error message.
         // Instead of setting no #title, form constructors are encouraged to set
@@ -1778,6 +1839,23 @@ function form_builder($form_id, &$element, &$form_state) {
     // from the POST data is set and matches the current form_id.
     if ($form_state['programmed'] || (!empty($form_state['input']) && (isset($form_state['input']['form_id']) && ($form_state['input']['form_id'] == $form_id)))) {
       $form_state['process_input'] = TRUE;
+      // If the session token was set by drupal_prepare_form(), ensure that it
+      // matches the current user's session.
+      $form_state['invalid_token'] = FALSE;
+      if (!empty($element['#token'])) {
+        if (empty($form_state['input']['form_token']) || !drupal_valid_token($form_state['input']['form_token'], $element['#token'])) {
+          // Set an early form error to block certain input processing since that
+          // opens the door for CSRF vulnerabilities.
+          _drupal_invalid_token_set_form_error();
+          // This value is checked in _form_builder_handle_input_element().
+          $form_state['invalid_token'] = TRUE;
+          // Ignore all submitted values.
+          $form_state['input'] = array();
+          $_POST = array();
+          // Make sure file uploads do not get processed.
+          $_FILES = array();
+        }
+      }
     }
     else {
       $form_state['process_input'] = FALSE;
@@ -1881,6 +1959,18 @@ function form_builder($form_id, &$element, &$form_state) {
       $element['#attributes']['enctype'] = 'multipart/form-data';
     }
 
+    // Allow Ajax submissions to the form action to bypass verification. This is
+    // especially useful for multipart forms, which cannot be verified via a
+    // response header.
+    $element['#attached']['js'][] = array(
+      'type' => 'setting',
+      'data' => array(
+        'urlIsAjaxTrusted' => array(
+          $element['#action'] => TRUE,
+        ),
+      ),
+    );
+
     // If a form contains a single textfield, and the ENTER key is pressed
     // within it, Internet Explorer submits the form with no POST data
     // identifying any submit button. Other browsers submit POST data as though
@@ -1929,6 +2019,19 @@ function form_builder($form_id, &$element, &$form_state) {
  * Adds the #name and #value properties of an input element before rendering.
  */
 function _form_builder_handle_input_element($form_id, &$element, &$form_state) {
+  static $safe_core_value_callbacks = array(
+    'form_type_token_value',
+    'form_type_textarea_value',
+    'form_type_textfield_value',
+    'form_type_checkbox_value',
+    'form_type_checkboxes_value',
+    'form_type_radios_value',
+    'form_type_password_confirm_value',
+    'form_type_select_value',
+    'form_type_tableselect_value',
+    'list_boolean_allowed_values_callback',
+  );
+
   if (!isset($element['#name'])) {
     $name = array_shift($element['#parents']);
     $element['#name'] = $name;
@@ -1978,7 +2081,7 @@ function _form_builder_handle_input_element($form_id, &$element, &$form_state) {
   // #access=FALSE on an element usually allow access for some users, so forms
   // submitted with drupal_form_submit() may bypass access restriction and be
   // treated as high-privilege users instead.
-  $process_input = empty($element['#disabled']) && ($form_state['programmed'] || ($form_state['process_input'] && (!isset($element['#access']) || $element['#access'])));
+  $process_input = empty($element['#disabled']) && (($form_state['programmed'] && $form_state['programmed_bypass_access_check']) || ($form_state['process_input'] && (!isset($element['#access']) || $element['#access'])));
 
   // Set the element's #value property.
   if (!isset($element['#value']) && !array_key_exists('#value', $element)) {
@@ -2007,7 +2110,14 @@ function _form_builder_handle_input_element($form_id, &$element, &$form_state) {
       // property, optionally filtered through $value_callback.
       if ($input_exists) {
         if (function_exists($value_callback)) {
-          $element['#value'] = $value_callback($element, $input, $form_state);
+          // Skip all value callbacks except safe ones like text if the CSRF
+          // token was invalid.
+          if (empty($form_state['invalid_token']) || in_array($value_callback, $safe_core_value_callbacks)) {
+            $element['#value'] = $value_callback($element, $input, $form_state);
+          }
+          else {
+            $input = NULL;
+          }
         }
         if (!isset($element['#value']) && isset($input)) {
           $element['#value'] = $input;
@@ -2402,6 +2512,17 @@ function form_type_password_confirm_value($element, $input = FALSE) {
     $element += array('#default_value' => array());
     return $element['#default_value'] + array('pass1' => '', 'pass2' => '');
   }
+  $value = array('pass1' => '', 'pass2' => '');
+  // Throw out all invalid array keys; we only allow pass1 and pass2.
+  foreach ($value as $allowed_key => $default) {
+    // These should be strings, but allow other scalars since they might be
+    // valid input in programmatic form submissions. Any nested array values
+    // are ignored.
+    if (isset($input[$allowed_key]) && is_scalar($input[$allowed_key])) {
+      $value[$allowed_key] = (string) $input[$allowed_key];
+    }
+  }
+  return $value;
 }
 
 /**
@@ -2445,6 +2566,27 @@ function form_type_select_value($element, $input = FALSE) {
   }
 }
 
+/**
+ * Determines the value for a textarea form element.
+ *
+ * @param array $element
+ *   The form element whose value is being populated.
+ * @param mixed $input
+ *   The incoming input to populate the form element. If this is FALSE,
+ *   the element's default value should be returned.
+ *
+ * @return string
+ *   The data that will appear in the $element_state['values'] collection
+ *   for this element. Return nothing to use the default.
+ */
+function form_type_textarea_value($element, $input = FALSE) {
+  if ($input !== FALSE && $input !== NULL) {
+    // This should be a string, but allow other scalars since they might be
+    // valid input in programmatic form submissions.
+    return is_scalar($input) ? (string) $input : '';
+  }
+}
+
 /**
  * Determines the value for a textfield form element.
  *
@@ -2460,9 +2602,12 @@ function form_type_select_value($element, $input = FALSE) {
  */
 function form_type_textfield_value($element, $input = FALSE) {
   if ($input !== FALSE && $input !== NULL) {
-    // Equate $input to the form value to ensure it's marked for
-    // validation.
-    return str_replace(array("\r", "\n"), '', $input);
+    // This should be a string, but allow other scalars since they might be
+    // valid input in programmatic form submissions.
+    if (!is_scalar($input)) {
+      $input = '';
+    }
+    return str_replace(array("\r", "\n"), '', (string) $input);
   }
 }
 
@@ -2578,8 +2723,8 @@ function _form_options_flatten($array) {
  *   - #required: (optional) Whether the user needs to select an option (TRUE)
  *     or not (FALSE). Defaults to FALSE.
  *   - #empty_option: (optional) The label to show for the first default option.
- *     By default, the label is automatically set to "- Please select -" for a
- *     required field and "- None -" for an optional field.
+ *     By default, the label is automatically set to "- Select -" for a required
+ *     field and "- None -" for an optional field.
  *   - #empty_value: (optional) The value for the first default option, which is
  *     used to determine whether the user submitted a value or not.
  *     - If #required is TRUE, this defaults to '' (an empty string).
@@ -2650,17 +2795,43 @@ function theme_select($variables) {
 }
 
 /**
- * Converts a select form element's options array into HTML.
- *
- * @param $element
- *   An associative array containing the properties of the element.
- * @param $choices
- *   Mixed: Either an associative array of items to list as choices, or an
- *   object with an 'option' member that is an associative array. This
- *   parameter is only used internally and should not be passed.
- *
- * @return
- *   An HTML string of options for the select form element.
+ * Converts an array of options into HTML, for use in select list form elements.
+ *
+ * This function calls itself recursively to obtain the values for each optgroup
+ * within the list of options and when the function encounters an object with
+ * an 'options' property inside $element['#options'].
+ *
+ * @param array $element
+ *   An associative array containing the following key-value pairs:
+ *   - #multiple: Optional Boolean indicating if the user may select more than
+ *     one item.
+ *   - #options: An associative array of options to render as HTML. Each array
+ *     value can be a string, an array, or an object with an 'option' property:
+ *     - A string or integer key whose value is a translated string is
+ *       interpreted as a single HTML option element. Do not use placeholders
+ *       that sanitize data: doing so will lead to double-escaping. Note that
+ *       the key will be visible in the HTML and could be modified by malicious
+ *       users, so don't put sensitive information in it.
+ *     - A translated string key whose value is an array indicates a group of
+ *       options. The translated string is used as the label attribute for the
+ *       optgroup. Do not use placeholders to sanitize data: doing so will lead
+ *       to double-escaping. The array should contain the options you wish to
+ *       group and should follow the syntax of $element['#options'].
+ *     - If the function encounters a string or integer key whose value is an
+ *       object with an 'option' property, the key is ignored, the contents of
+ *       the option property are interpreted as $element['#options'], and the
+ *       resulting HTML is added to the output.
+ *   - #value: Optional integer, string, or array representing which option(s)
+ *     to pre-select when the list is first displayed. The integer or string
+ *     must match the key of an option in the '#options' list. If '#multiple' is
+ *     TRUE, this can be an array of integers or strings.
+ * @param array|null $choices
+ *   (optional) Either an associative array of options in the same format as
+ *   $element['#options'] above, or NULL. This parameter is only used internally
+ *   and is not intended to be passed in to the initial function call.
+ *
+ * @return string
+ *   An HTML string of options and optgroups for use in a select form element.
  */
 function form_select_options($element, $choices = NULL) {
   if (!isset($choices)) {
@@ -2673,7 +2844,7 @@ function form_select_options($element, $choices = NULL) {
   $options = '';
   foreach ($choices as $key => $choice) {
     if (is_array($choice)) {
-      $options .= '<optgroup label="' . $key . '">';
+      $options .= '<optgroup label="' . check_plain($key) . '">';
       $options .= form_select_options($element, $choice);
       $options .= '</optgroup>';
     }
@@ -2866,7 +3037,7 @@ function form_process_password_confirm($element) {
 function password_confirm_validate($element, &$element_state) {
   $pass1 = trim($element['pass1']['#value']);
   $pass2 = trim($element['pass2']['#value']);
-  if (!empty($pass1) || !empty($pass2)) {
+  if (strlen($pass1) > 0 || strlen($pass2) > 0) {
     if (strcmp($pass1, $pass2)) {
       form_error($element, t('The specified passwords do not match.'));
     }
@@ -3051,8 +3222,7 @@ function form_process_radios($element) {
  * @param $variables
  *   An associative array containing:
  *   - element: An associative array containing the properties of the element.
- *     Properties used: #title, #value, #return_value, #description, #required,
- *     #attributes, #checked.
+ *     Properties used: #id, #name, #attributes, #checked, #return_value.
  *
  * @ingroup themeable
  */
@@ -3224,9 +3394,12 @@ function form_process_container($element, &$form_state) {
 /**
  * Returns HTML to wrap child elements in a container.
  *
- * Used for grouped form items. Can also be used as a #theme_wrapper for any
+ * Used for grouped form items. Can also be used as a theme wrapper for any
  * renderable element, to surround it with a <div> and add attributes such as
- * classes or an HTML id.
+ * classes or an HTML ID.
+ *
+ * See the @link forms_api_reference.html Form API reference @endlink for more
+ * information on the #theme_wrappers render array property.
  *
  * @param $variables
  *   An associative array containing:
@@ -3237,6 +3410,8 @@ function form_process_container($element, &$form_state) {
  */
 function theme_container($variables) {
   $element = $variables['element'];
+  // Ensure #attributes is set.
+  $element += array('#attributes' => array());
 
   // Special handling for form elements.
   if (isset($element['#array_parents'])) {
@@ -3379,6 +3554,7 @@ function form_process_tableselect($element) {
             '#return_value' => $key,
             '#default_value' => isset($value[$key]) ? $key : NULL,
             '#attributes' => $element['#attributes'],
+            '#ajax' => isset($element['#ajax']) ? $element['#ajax'] : NULL,
           );
         }
         else {
@@ -3799,6 +3975,34 @@ function theme_hidden($variables) {
   return '<input' . drupal_attributes($element['#attributes']) . " />\n";
 }
 
+/**
+ * Process function to prepare autocomplete data.
+ *
+ * @param $element
+ *   A textfield or other element with a #autocomplete_path.
+ *
+ * @return array
+ *   The processed form element.
+ */
+function form_process_autocomplete($element) {
+  $element['#autocomplete_input'] = array();
+  if ($element['#autocomplete_path'] && drupal_valid_path($element['#autocomplete_path'])) {
+    $element['#autocomplete_input']['#id'] = $element['#id'] .'-autocomplete';
+    // Force autocomplete to use non-clean URLs since this protects against the
+    // browser interpreting the path plus search string as an actual file.
+    $current_clean_url = isset($GLOBALS['conf']['clean_url']) ? $GLOBALS['conf']['clean_url'] : NULL;
+    $GLOBALS['conf']['clean_url'] = 0;
+    // Force the script path to 'index.php', in case the server is not
+    // configured to find it automatically. Normally it is the responsibility
+    // of the site to do this themselves using hook_url_outbound_alter() (see
+    // url()) but since this code is forcing non-clean URLs on sites that don't
+    // normally use them, it is done here instead.
+    $element['#autocomplete_input']['#url_value'] = url($element['#autocomplete_path'], array('absolute' => TRUE, 'script' => 'index.php'));
+    $GLOBALS['conf']['clean_url'] = $current_clean_url;
+  }
+  return $element;
+}
+
 /**
  * Returns HTML for a textfield form element.
  *
@@ -3817,14 +4021,14 @@ function theme_textfield($variables) {
   _form_set_class($element, array('form-text'));
 
   $extra = '';
-  if ($element['#autocomplete_path'] && drupal_valid_path($element['#autocomplete_path'])) {
+  if ($element['#autocomplete_path'] && !empty($element['#autocomplete_input'])) {
     drupal_add_library('system', 'drupal.autocomplete');
     $element['#attributes']['class'][] = 'form-autocomplete';
 
     $attributes = array();
     $attributes['type'] = 'hidden';
-    $attributes['id'] = $element['#attributes']['id'] . '-autocomplete';
-    $attributes['value'] = url($element['#autocomplete_path'], array('absolute' => TRUE));
+    $attributes['id'] = $element['#autocomplete_input']['#id'];
+    $attributes['value'] = $element['#autocomplete_input']['#url_value'];
     $attributes['disabled'] = 'disabled';
     $attributes['class'][] = 'autocomplete';
     $extra = '<input' . drupal_attributes($attributes) . ' />';
@@ -4244,7 +4448,7 @@ function element_validate_number($element, &$form_state) {
  * returns any user input in the 'results' or 'message' keys of $context,
  * it must also sanitize them first.
  *
- * Sample batch operations:
+ * Sample callback_batch_operation():
  * @code
  * // Simple and artificial: load a node of a given type for a given user
  * function my_function_1($uid, $type, &$context) {
@@ -4296,9 +4500,9 @@ function element_validate_number($element, &$form_state) {
  * }
  * @endcode
  *
- * Sample 'finished' callback:
+ * Sample callback_batch_finished():
  * @code
- * function batch_test_finished($success, $results, $operations) {
+ * function my_finished_callback($success, $results, $operations) {
  *   // The 'success' parameter means no fatal PHP errors were detected. All
  *   // other error management should be handled using 'results'.
  *   if ($success) {
@@ -4335,12 +4539,14 @@ function element_validate_number($element, &$form_state) {
  * @param $batch_definition
  *   An associative array defining the batch, with the following elements (all
  *   are optional except as noted):
- *   - operations: (required) Array of function calls to be performed.
+ *   - operations: (required) Array of operations to be performed, where each
+ *     item is an array consisting of the name of an implementation of
+ *     callback_batch_operation() and an array of parameter.
  *     Example:
  *     @code
  *     array(
- *       array('my_function_1', array($arg1)),
- *       array('my_function_2', array($arg2_1, $arg2_2)),
+ *       array('callback_batch_operation_1', array($arg1)),
+ *       array('callback_batch_operation_2', array($arg2_1, $arg2_2)),
  *     )
  *     @endcode
  *   - title: A safe, translated string to use as the title for the progress
@@ -4352,10 +4558,10 @@ function element_validate_number($element, &$form_state) {
  *     @elapsed. Defaults to t('Completed @current of @total.').
  *   - error_message: Message displayed if an error occurred while processing
  *     the batch. Defaults to t('An error has occurred.').
- *   - finished: Name of a function to be executed after the batch has
- *     completed. This should be used to perform any result massaging that may
- *     be needed, and possibly save data in $_SESSION for display after final
- *     page redirection.
+ *   - finished: Name of an implementation of callback_batch_finished(). This is
+ *     executed after the batch has completed. This should be used to perform
+ *     any result massaging that may be needed, and possibly save data in
+ *     $_SESSION for display after final page redirection.
  *   - file: Path to the file containing the definitions of the 'operations' and
  *     'finished' functions, for instance if they don't reside in the main
  *     .module file. The path should be relative to base_path(), and thus should

includes/install.core.inc

@@ -362,7 +362,8 @@ function install_run_tasks(&$install_state) {
  * Runs an individual installation task.
  *
  * @param $task
- *   An array of information about the task to be run.
+ *   An array of information about the task to be run as returned by
+ *   hook_install_tasks().
  * @param $install_state
  *   An array of information about the current installation state. This is
  *   passed in by reference so that it can be modified by the task.
@@ -478,11 +479,15 @@ function install_run_task($task, &$install_state) {
  * the page request evolves (for example, if an installation profile hasn't
  * been selected yet, we don't yet know which profile tasks need to be run).
  *
+ * You can override this using hook_install_tasks() or
+ * hook_install_tasks_alter().
+ *
  * @param $install_state
  *   An array of information about the current installation state.
  *
  * @return
- *   A list of tasks to be performed, with associated metadata.
+ *   A list of tasks to be performed, with associated metadata as returned by
+ *   hook_install_tasks().
  */
 function install_tasks_to_perform($install_state) {
   // Start with a list of all currently available tasks.
@@ -692,6 +697,21 @@ function install_full_redirect_url($install_state) {
  */
 function install_display_output($output, $install_state) {
   drupal_page_header();
+
+  // Prevent install.php from being indexed when installed in a sub folder.
+  // robots.txt rules are not read if the site is within domain.com/subfolder
+  // resulting in /subfolder/install.php being found through search engines.
+  // When settings.php is writeable this can be used via an external database
+  // leading a malicious user to gain php access to the server.
+  $noindex_meta_tag = array(
+    '#tag' => 'meta',
+    '#attributes' => array(
+      'name' => 'robots',
+      'content' => 'noindex, nofollow',
+    ),
+  );
+  drupal_add_html_head($noindex_meta_tag, 'install_meta_robots');
+
   // Only show the task list if there is an active task; otherwise, the page
   // request has ended before tasks have even been started, so there is nothing
   // meaningful to show.
@@ -766,6 +786,15 @@ function install_system_module(&$install_state) {
   // Install system.module.
   drupal_install_system();
 
+  // Call file_ensure_htaccess() to ensure that all of Drupal's standard
+  // directories (e.g., the public and private files directories) have
+  // appropriate .htaccess files. These directories will have already been
+  // created by this point in the installer, since Drupal creates them during
+  // the install_verify_requirements() task. Note that we cannot call
+  // file_ensure_htaccess() any earlier than this, since it relies on
+  // system.module in order to work.
+  file_ensure_htaccess();
+
   // Enable the user module so that sessions can be recorded during the
   // upcoming bootstrap step.
   module_enable(array('user'), FALSE);
@@ -780,6 +809,13 @@ function install_system_module(&$install_state) {
 
   variable_set('install_profile_modules', array_diff($modules, array('system')));
   $install_state['database_tables_exist'] = TRUE;
+
+  // Prevent the hook_requirements() check from telling us to convert the
+  // database to utf8mb4.
+  $connection = Database::getConnection();
+  if ($connection->utf8mb4IsConfigurable() && $connection->utf8mb4IsActive()) {
+    variable_set('drupal_all_databases_are_utf8mb4', TRUE);
+  }
 }
 
 /**
@@ -981,7 +1017,7 @@ function install_settings_form_submit($form, &$form_state) {
     'required' => TRUE,
   );
   $settings['drupal_hash_salt'] = array(
-    'value'    => drupal_hash_base64(drupal_random_bytes(55)),
+    'value'    => drupal_random_key(),
     'required' => TRUE,
   );
   drupal_rewrite_settings($settings);
@@ -1561,7 +1597,9 @@ function install_finished(&$install_state) {
 }
 
 /**
- * Batch callback for batch installation of modules.
+ * Implements callback_batch_operation().
+ *
+ * Performs batch installation of modules.
  */
 function _install_module_batch($module, $module_name, &$context) {
   // Install and enable the module right away, so that the module will be
@@ -1574,6 +1612,8 @@ function _install_module_batch($module, $module_name, &$context) {
 }
 
 /**
+ * Implements callback_batch_finished().
+ *
  * 'Finished' callback for module installation batch.
  */
 function _install_profile_modules_finished($success, $results, $operations) {

includes/install.inc

@@ -420,7 +420,7 @@ abstract class DatabaseTasks {
       }
     }
     if (!empty($message)) {
-      $message = '<p>In order for Drupal to work, and to continue with the installation process, you must resolve all issues reported below. For more help with configuring your database server, see the <a href="http://drupal.org/getting-started/install">installation handbook</a>. If you are unsure what any of this means you should probably contact your hosting provider.</p>' . $message;
+      $message = 'Resolve all issues below to continue the installation. For help configuring your database server, see the <a href="http://drupal.org/getting-started/install">installation handbook</a>, or contact your hosting provider.' . $message;
       throw new DatabaseTaskException($message);
     }
   }
@@ -653,6 +653,13 @@ function drupal_rewrite_settings($settings = array(), $prefix = '') {
     if ($fp && fwrite($fp, $buffer) === FALSE) {
       throw new Exception(st('Failed to modify %settings. Verify the file permissions.', array('%settings' => $settings_file)));
     }
+    else {
+      // The existing settings.php file might have been included already. In
+      // case an opcode cache is enabled, the rewritten contents of the file
+      // will not be reflected in this process. Ensure to invalidate the file
+      // in case an opcode cache is enabled.
+      drupal_clear_opcode_cache(DRUPAL_ROOT . '/' . $settings_file);
+    }
   }
   else {
     throw new Exception(st('Failed to open %settings. Verify the file permissions.', array('%settings' => $default_settings)));
@@ -741,20 +748,28 @@ function drupal_install_system() {
 }
 
 /**
- * Uninstalls a given list of modules.
+ * Uninstalls a given list of disabled modules.
  *
- * @param $module_list
- *   The modules to uninstall.
- * @param $uninstall_dependents
- *   If TRUE, the function will check that all modules which depend on the
- *   passed-in module list either are already uninstalled or contained in the
- *   list, and it will ensure that the modules are uninstalled in the correct
- *   order. This incurs a significant performance cost, so use FALSE if you
- *   know $module_list is already complete and in the correct order.
+ * @param string[] $module_list
+ *   The modules to uninstall. It is the caller's responsibility to ensure that
+ *   all modules in this list have already been disabled before this function
+ *   is called.
+ * @param bool $uninstall_dependents
+ *   (optional) If TRUE, the function will check that all modules which depend
+ *   on the passed-in module list either are already uninstalled or contained in
+ *   the list, and it will ensure that the modules are uninstalled in the
+ *   correct order. This incurs a significant performance cost, so use FALSE if
+ *   you know $module_list is already complete and in the correct order.
+ *   Defaults to TRUE.
  *
- * @return
- *   FALSE if one or more dependent modules are missing from the list, TRUE
- *   otherwise.
+ * @return bool
+ *   Returns TRUE if the operation succeeds or FALSE if it aborts due to an
+ *   unsafe condition, namely, $uninstall_dependents is TRUE and a module in
+ *   $module_list has dependents which are not already uninstalled and not also
+ *   included in $module_list).
+ *
+ * @see module_disable()
+ * @see module_enable()
  */
 function drupal_uninstall_modules($module_list = array(), $uninstall_dependents = TRUE) {
   if ($uninstall_dependents) {
@@ -764,9 +779,9 @@ function drupal_uninstall_modules($module_list = array(), $uninstall_dependents
     $module_list = array_flip(array_values($module_list));
 
     $profile = drupal_get_profile();
-    while (list($module) = each($module_list)) {
+    foreach (array_keys($module_list) as $module) {
       if (!isset($module_data[$module]) || drupal_get_installed_schema_version($module) == SCHEMA_UNINSTALLED) {
-        // This module doesn't exist or is already uninstalled, skip it.
+        // This module doesn't exist or is already uninstalled. Skip it.
         unset($module_list[$module]);
         continue;
       }
@@ -799,7 +814,7 @@ function drupal_uninstall_modules($module_list = array(), $uninstall_dependents
   }
 
   if (!empty($module_list)) {
-    // Call hook_module_uninstall to let other modules act
+    // Let other modules react.
     module_invoke_all('modules_uninstalled', $module_list);
   }
 
@@ -1127,7 +1142,6 @@ function st($string, array $args = array(), array $options = array()) {
     }
   }
 
-  require_once DRUPAL_ROOT . '/includes/theme.inc';
   // Transform arguments before inserting them
   foreach ($args as $key => $value) {
     switch ($key[0]) {

includes/iso.inc

@@ -53,6 +53,7 @@ function _country_get_predefined_list() {
     'BM' => $t('Bermuda'),
     'BN' => $t('Brunei'),
     'BO' => $t('Bolivia'),
+    'BQ' => $t('Caribbean Netherlands'),
     'BR' => $t('Brazil'),
     'BS' => $t('Bahamas'),
     'BT' => $t('Bhutan'),
@@ -74,8 +75,8 @@ function _country_get_predefined_list() {
     'CO' => $t('Colombia'),
     'CR' => $t('Costa Rica'),
     'CU' => $t('Cuba'),
-    'CW' => $t('Curaçao'),
     'CV' => $t('Cape Verde'),
+    'CW' => $t('Curaçao'),
     'CX' => $t('Christmas Island'),
     'CY' => $t('Cyprus'),
     'CZ' => $t('Czech Republic'),
@@ -230,8 +231,10 @@ function _country_get_predefined_list() {
     'SN' => $t('Senegal'),
     'SO' => $t('Somalia'),
     'SR' => $t('Suriname'),
+    'SS' => $t('South Sudan'),
     'ST' => $t('Sao Tome and Principe'),
     'SV' => $t('El Salvador'),
+    'SX' => $t('Sint Maarten'),
     'SY' => $t('Syria'),
     'SZ' => $t('Swaziland'),
     'TC' => $t('Turks and Caicos Islands'),

includes/language.inc

@@ -78,7 +78,7 @@ define('LANGUAGE_NEGOTIATION_DEFAULT', 'language-default');
  * function mymodule_language_negotiation_info_alter(&$negotiation_info) {
  *   // Replace the core function with our own function.
  *   module_load_include('language', 'inc', 'language.negotiation');
- *   $negotiation_info[LANGUAGE_NEGOTIATION_URL]['callbacks']['negotiation'] = 'mymodule_from_url';
+ *   $negotiation_info[LANGUAGE_NEGOTIATION_URL]['callbacks']['language'] = 'mymodule_from_url';
  *   $negotiation_info[LANGUAGE_NEGOTIATION_URL]['file'] = drupal_get_path('module', 'mymodule') . '/mymodule.module';
  * }
  *
@@ -94,7 +94,6 @@ define('LANGUAGE_NEGOTIATION_DEFAULT', 'language-default');
  *   }
  *   return $langcode;
  * }
- * ?>
  * @endcode
  *
  * For more information, see
@@ -298,7 +297,7 @@ function language_negotiation_get_switch_links($type, $path) {
       // Add support for WCAG 2.0's Language of Parts to add language identifiers.
       // http://www.w3.org/TR/UNDERSTANDING-WCAG20/meaning-other-lang-id.html
       foreach ($result as $langcode => $link) {
-        $result[$langcode]['attributes']['lang'] = $langcode;
+        $result[$langcode]['attributes']['xml:lang'] = $langcode;
       }
 
       if (!empty($result)) {
@@ -314,7 +313,7 @@ function language_negotiation_get_switch_links($type, $path) {
 }
 
 /**
- * Removes any unused language negotation providers from the configuration.
+ * Removes any unused language negotiation providers from the configuration.
  */
 function language_negotiation_purge() {
   // Ensure that we are getting the defined language negotiation information. An

includes/locale.inc

@@ -398,7 +398,7 @@ function locale_language_switcher_session($type, $path) {
       $links[$langcode]['query'][$param] = $langcode;
     }
     else {
-      $links[$langcode]['attributes']['class'][] = ' session-active';
+      $links[$langcode]['attributes']['class'][] = 'session-active';
     }
   }
 
@@ -435,6 +435,13 @@ function locale_language_url_rewrite_url(&$path, &$options) {
     switch (variable_get('locale_language_negotiation_url_part', LOCALE_LANGUAGE_NEGOTIATION_URL_PREFIX)) {
       case LOCALE_LANGUAGE_NEGOTIATION_URL_DOMAIN:
         if ($options['language']->domain) {
+          // Save the original base URL. If it contains a port, we need to
+          // retain it below.
+          if (!empty($options['base_url'])) {
+            // The colon in the URL scheme messes up the port checking below.
+            $normalized_base_url = str_replace(array('https://', 'http://'), '', $options['base_url']);
+          }
+
           // Ask for an absolute URL with our modified base_url.
           global $is_https;
           $url_scheme = ($is_https) ? 'https://' : 'http://';
@@ -449,6 +456,19 @@ function locale_language_url_rewrite_url(&$path, &$options) {
 
           // Apply the appropriate protocol to the URL.
           $options['base_url'] = $url_scheme . $host;
+
+          // In case either the original base URL or the HTTP host contains a
+          // port, retain it.
+          $http_host = $_SERVER['HTTP_HOST'];
+          if (isset($normalized_base_url) && strpos($normalized_base_url, ':') !== FALSE) {
+            list($host, $port) = explode(':', $normalized_base_url);
+            $options['base_url'] .= ':' . $port;
+          }
+          elseif (strpos($http_host, ':') !== FALSE) {
+            list($host, $port) = explode(':', $http_host);
+            $options['base_url'] .= ':' . $port;
+          }
+
           if (isset($options['https']) && variable_get('https', FALSE)) {
             if ($options['https'] === TRUE) {
               $options['base_url'] = str_replace('http://', 'https://', $options['base_url']);
@@ -523,6 +543,22 @@ function locale_language_url_rewrite_session(&$path, &$options) {
  * possible attack vector (img).
  */
 function locale_string_is_safe($string) {
+  // Some strings have tokens in them. For tokens in the first part of href or
+  // src HTML attributes, filter_xss() removes part of the token, the part
+  // before the first colon.  filter_xss() assumes it could be an attempt to
+  // inject javascript. When filter_xss() removes part of tokens, it causes the
+  // string to not be translatable when it should be translatable. See
+  // LocaleStringIsSafeTest::testLocaleStringIsSafe().
+  //
+  // We can recognize tokens since they are wrapped with brackets and are only
+  // composed of alphanumeric characters, colon, underscore, and dashes. We can
+  // be sure these strings are safe to strip out before the string is checked in
+  // filter_xss() because no dangerous javascript will match that pattern.
+  //
+  // @todo Do not strip out the token. Fix filter_xss() to not incorrectly
+  //   alter the string. https://www.drupal.org/node/2372127
+  $string = preg_replace('/\[[a-z0-9_-]+(:[a-z0-9_-]+)+\]/i', '', $string);
+
   return decode_entities($string) == decode_entities(filter_xss($string, array('a', 'abbr', 'acronym', 'address', 'b', 'bdo', 'big', 'blockquote', 'br', 'caption', 'cite', 'code', 'col', 'colgroup', 'dd', 'del', 'dfn', 'dl', 'dt', 'em', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'i', 'ins', 'kbd', 'li', 'ol', 'p', 'pre', 'q', 'samp', 'small', 'span', 'strong', 'sub', 'sup', 'table', 'tbody', 'td', 'tfoot', 'th', 'thead', 'tr', 'tt', 'ul', 'var')));
 }
 
@@ -631,9 +667,6 @@ function locale_add_language($langcode, $name = NULL, $native = NULL, $direction
  *   translations).
  */
 function _locale_import_po($file, $langcode, $mode, $group = NULL) {
-  // Try to allocate enough time to parse and import the data.
-  drupal_set_time_limit(240);
-
   // Check if we have the language already in the database.
   if (!db_query("SELECT COUNT(language) FROM {languages} WHERE language = :language", array(':language' => $langcode))->fetchField()) {
     drupal_set_message(t('The language selected for import is not supported.'), 'error');
@@ -717,6 +750,12 @@ function _locale_import_read_po($op, $file, $mode = NULL, $lang = NULL, $group =
   $lineno = 0;
 
   while (!feof($fd)) {
+    // Refresh the time limit every 10 parsed rows to ensure there is always
+    // enough time to import the data for large PO files.
+    if (!($lineno % 10)) {
+      drupal_set_time_limit(30);
+    }
+
     // A line should not be longer than 10 * 1024.
     $line = fgets($fd, 10 * 1024);
 
@@ -1931,7 +1970,7 @@ function _locale_translate_seek() {
       $groups[$string['group']],
       array('data' => check_plain(truncate_utf8($string['source'], 150, FALSE, TRUE)) . '<br /><small>' . $string['location'] . '</small>'),
       $string['context'],
-      array('data' => _locale_translate_language_list($string['languages'], $limit_language), 'align' => 'center'),
+      array('data' => _locale_translate_language_list($string, $limit_language), 'align' => 'center'),
       array('data' => l(t('edit'), "admin/config/regional/translate/edit/$lid", array('query' => drupal_get_destination())), 'class' => array('nowrap')),
       array('data' => l(t('delete'), "admin/config/regional/translate/delete/$lid", array('query' => drupal_get_destination())), 'class' => array('nowrap')),
     );
@@ -2126,16 +2165,21 @@ function _locale_rebuild_js($langcode = NULL) {
 /**
  * List languages in search result table
  */
-function _locale_translate_language_list($translation, $limit_language) {
+function _locale_translate_language_list($string, $limit_language) {
   // Add CSS.
   drupal_add_css(drupal_get_path('module', 'locale') . '/locale.css');
 
+  // Include both translated and not yet translated target languages in the
+  // list. The source language is English for built-in strings and the default
+  // language for other strings.
   $languages = language_list();
-  unset($languages['en']);
+  $default = language_default();
+  $omit = $string['group'] == 'default' ? 'en' : $default->language;
+  unset($languages[$omit]);
   $output = '';
   foreach ($languages as $langcode => $language) {
     if (!$limit_language || $limit_language == $langcode) {
-      $output .= (!empty($translation[$langcode])) ? $langcode . ' ' : "<em class=\"locale-untranslated\">$langcode</em> ";
+      $output .= (!empty($string['languages'][$langcode])) ? $langcode . ' ' : "<em class=\"locale-untranslated\">$langcode</em> ";
     }
   }
 
@@ -2301,6 +2345,8 @@ function _locale_batch_build($files, $finished = NULL, $components = array()) {
 }
 
 /**
+ * Implements callback_batch_operation().
+ *
  * Perform interface translation import as a batch step.
  *
  * @param $filepath
@@ -2319,6 +2365,8 @@ function _locale_batch_import($filepath, &$context) {
 }
 
 /**
+ * Implements callback_batch_finished().
+ *
  * Finished callback of system page locale import batch.
  * Inform the user of translation files imported.
  */
@@ -2329,6 +2377,8 @@ function _locale_batch_system_finished($success, $results) {
 }
 
 /**
+ * Implements callback_batch_finished().
+ *
  * Finished callback of language addition locale import batch.
  * Inform the user of translation files imported.
  */

includes/lock.inc

@@ -92,7 +92,7 @@ function _lock_id() {
  * Acquire (or renew) a lock, but do not block if it fails.
  *
  * @param $name
- *   The name of the lock.
+ *   The name of the lock. Limit of name's length is 255 characters.
  * @param $timeout
  *   A number of seconds (float) before the lock expires (minimum of 0.001).
  *

includes/mail.inc

@@ -10,7 +10,7 @@
  *
  * $conf['mail_line_endings'] will override this setting.
  */
-define('MAIL_LINE_ENDINGS', isset($_SERVER['WINDIR']) || strpos($_SERVER['SERVER_SOFTWARE'], 'Win32') !== FALSE ? "\r\n" : "\n");
+define('MAIL_LINE_ENDINGS', isset($_SERVER['WINDIR']) || (isset($_SERVER['SERVER_SOFTWARE']) && strpos($_SERVER['SERVER_SOFTWARE'], 'Win32') !== FALSE) ? "\r\n" : "\n");
 
 /**
  * Composes and optionally sends an e-mail message.
@@ -339,13 +339,13 @@ interface MailSystemInterface {
  *
  * We deliberately use LF rather than CRLF, see drupal_mail().
  *
- * @param $text
+ * @param string $text
  *   The plain text to process.
- * @param $indent (optional)
+ * @param string $indent (optional)
  *   A string to indent the text with. Only '>' characters are repeated on
  *   subsequent wrapped lines. Others are replaced by spaces.
  *
- * @return
+ * @return string
  *   The content of the email as a string with formatting applied.
  */
 function drupal_wrap_mail($text, $indent = '') {
@@ -356,8 +356,9 @@ function drupal_wrap_mail($text, $indent = '') {
   $soft = strpos($clean_indent, ' ') === FALSE;
   // Check if the string has line breaks.
   if (strpos($text, "\n") !== FALSE) {
-    // Remove trailing spaces to make existing breaks hard.
-    $text = preg_replace('/ +\n/m', "\n", $text);
+    // Remove trailing spaces to make existing breaks hard, but leave signature
+    // marker untouched (RFC 3676, Section 4.3).
+    $text = preg_replace('/(?(?<!^--) +\n|  +\n)/m', "\n", $text);
     // Wrap each line at the needed width.
     $lines = explode("\n", $text);
     array_walk($lines, '_drupal_wrap_mail_line', array('soft' => $soft, 'length' => strlen($indent)));
@@ -563,9 +564,9 @@ function drupal_html_to_text($string, $allowed_tags = NULL) {
  */
 function _drupal_wrap_mail_line(&$line, $key, $values) {
   // Use soft-breaks only for purely quoted or unindented text.
-  $line = wordwrap($line, 77 - $values['length'], $values['soft'] ? "  \n" : "\n");
+  $line = wordwrap($line, 77 - $values['length'], $values['soft'] ? " \n" : "\n");
   // Break really long words at the maximum width allowed.
-  $line = wordwrap($line, 996 - $values['length'], $values['soft'] ? " \n" : "\n");
+  $line = wordwrap($line, 996 - $values['length'], $values['soft'] ? " \n" : "\n", TRUE);
 }
 
 /**

includes/menu.inc

@@ -229,12 +229,20 @@ define('MENU_CONTEXT_INLINE', 0x0002);
 define('MENU_FOUND', 1);
 
 /**
- * Internal menu status code -- Menu item was not found.
+ * Menu status code -- Not found.
+ *
+ * This can be used as the return value from a page callback, although it is
+ * preferable to use a load function to accomplish this; see the hook_menu()
+ * documentation for details.
  */
 define('MENU_NOT_FOUND', 2);
 
 /**
- * Internal menu status code -- Menu item access is denied.
+ * Menu status code -- Access denied.
+ *
+ * This can be used as the return value from a page callback, although it is
+ * preferable to use an access callback to accomplish this; see the hook_menu()
+ * documentation for details.
  */
 define('MENU_ACCESS_DENIED', 3);
 
@@ -309,7 +317,7 @@ define('MENU_PREFERRED_LINK', '1cf698d64d1aa4b83907cf6ed55db3a7f8e92c91');
  * actually exists. This list of 'masks' is built in menu_rebuild().
  *
  * @param $parts
- *   An array of path parts, for the above example
+ *   An array of path parts; for the above example, 
  *   array('node', '12345', 'edit').
  *
  * @return
@@ -430,8 +438,8 @@ function menu_set_item($path, $router_item) {
  * Gets a router item.
  *
  * @param $path
- *   The path, for example node/5. The function will find the corresponding
- *   node/% item and return that.
+ *   The path; for example, 'node/5'. The function will find the corresponding
+ *   node/% item and return that. Defaults to the current path.
  * @param $router_item
  *   Internal use only.
  *
@@ -456,7 +464,9 @@ function menu_get_item($path = NULL, $router_item = NULL) {
     // Rebuild if we know it's needed, or if the menu masks are missing which
     // occurs rarely, likely due to a race condition of multiple rebuilds.
     if (variable_get('menu_rebuild_needed', FALSE) || !variable_get('menu_masks', array())) {
-      menu_rebuild();
+      if (_menu_check_rebuild()) {
+        menu_rebuild();
+      }
     }
     $original_map = arg(NULL, $path);
 
@@ -542,7 +552,7 @@ function menu_execute_active_handler($path = NULL, $deliver = TRUE) {
  * @param $item
  *   A menu router or menu link item
  * @param $map
- *   An array of path arguments (ex: array('node', '5'))
+ *   An array of path arguments; for example, array('node', '5').
  *
  * @return
  *   Returns TRUE for success, FALSE if an object cannot be loaded.
@@ -566,7 +576,8 @@ function _menu_load_objects(&$item, &$map) {
           // 'load arguments' in the hook_menu() entry, but they need
           // some processing. In this case the $function is the key to the
           // load_function array, and the value is the list of arguments.
-          list($function, $args) = each($function);
+          $args = current($function);
+          $function = key($function);
           $load_functions[$index] = $function;
 
           // Some arguments are placeholders for dynamic items to process.
@@ -612,7 +623,7 @@ function _menu_load_objects(&$item, &$map) {
  * @param $item
  *   A menu router or menu link item
  * @param $map
- *   An array of path arguments (ex: array('node', '5'))
+ *   An array of path arguments; for example, array('node', '5').
  *
  * @return
  *   $item['access'] becomes TRUE if the item is accessible, FALSE otherwise.
@@ -738,7 +749,7 @@ function _menu_item_localize(&$item, $map, $link_translate = FALSE) {
  * @param $router_item
  *   A menu router item
  * @param $map
- *   An array of path arguments (ex: array('node', '5'))
+ *   An array of path arguments; for example, array('node', '5').
  * @param $to_arg
  *   Execute $item['to_arg_functions'] or not. Use only if you want to render a
  *   path from the menu table, for example tabs.
@@ -801,9 +812,9 @@ function _menu_translate(&$router_item, $map, $to_arg = FALSE) {
  * Translates the path elements in the map using any to_arg helper function.
  *
  * @param $map
- *   An array of path arguments (ex: array('node', '5'))
+ *   An array of path arguments; for example, array('node', '5').
  * @param $to_arg_functions
- *   An array of helper function (ex: array(2 => 'menu_tail_to_arg'))
+ *   An array of helper functions; for example, array(2 => 'menu_tail_to_arg').
  *
  * @see hook_menu()
  */
@@ -1000,7 +1011,7 @@ function menu_tree($menu_name) {
 }
 
 /**
- * Returns a rendered menu tree.
+ * Returns an output structure for rendering a menu tree.
  *
  * The menu item's LI element is given one of the following classes:
  * - expanded: The menu item is showing its submenu.
@@ -1485,7 +1496,7 @@ function menu_tree_collect_node_links(&$tree, &$node_links) {
  *   menu_tree_collect_node_links().
  */
 function menu_tree_check_access(&$tree, $node_links = array()) {
-  if ($node_links) {
+  if ($node_links && (user_access('access content') || user_access('bypass node access'))) {
     $nids = array_keys($node_links);
     $select = db_select('node', 'n');
     $select->addField('n', 'nid');
@@ -1596,6 +1607,7 @@ function _menu_tree_data(&$links, $parents, $depth) {
  * Implements template_preprocess_HOOK() for theme_menu_tree().
  */
 function template_preprocess_menu_tree(&$variables) {
+  $variables['#tree'] = $variables['tree'];
   $variables['tree'] = $variables['tree']['#children'];
 }
 
@@ -1926,13 +1938,21 @@ function menu_local_tasks($level = 0) {
     }
 
     // Get all tabs (also known as local tasks) and the root page.
-    $result = db_select('menu_router', NULL, array('fetch' => PDO::FETCH_ASSOC))
-      ->fields('menu_router')
-      ->condition('tab_root', $router_item['tab_root'])
-      ->condition('context', MENU_CONTEXT_INLINE, '<>')
-      ->orderBy('weight')
-      ->orderBy('title')
-      ->execute();
+    $cid = 'local_tasks:' . $router_item['tab_root'];
+    if ($cache = cache_get($cid, 'cache_menu')) {
+      $result = $cache->data;
+    }
+    else {
+      $result = db_select('menu_router', NULL, array('fetch' => PDO::FETCH_ASSOC))
+        ->fields('menu_router')
+        ->condition('tab_root', $router_item['tab_root'])
+        ->condition('context', MENU_CONTEXT_INLINE, '<>')
+        ->orderBy('weight')
+        ->orderBy('title')
+        ->execute()
+        ->fetchAll();
+      cache_set($cid, $result, 'cache_menu');
+    }
     $map = $router_item['original_map'];
     $children = array();
     $tasks = array();
@@ -2139,7 +2159,7 @@ function menu_local_tasks($level = 0) {
  *   example 'node' or 'admin/structure/block/manage'.
  * @param $args
  *   A list of dynamic path arguments to append to $parent_path to form the
- *   fully-qualified menu router path, for example array(123) for a certain
+ *   fully-qualified menu router path; for example, array(123) for a certain
  *   node or array('system', 'navigation') for a certain block.
  *
  * @return
@@ -2383,7 +2403,8 @@ function menu_set_active_trail($new_trail = NULL) {
       // a stripped down menu tree containing the active trail only, in case
       // the given menu has not been built in this request yet.
       $tree = menu_tree_page_data($preferred_link['menu_name'], NULL, TRUE);
-      list($key, $curr) = each($tree);
+      $curr = current($tree);
+      next($tree);
     }
     // There is no link for the current path.
     else {
@@ -2401,7 +2422,7 @@ function menu_set_active_trail($new_trail = NULL) {
           // argument placeholders (%). Such links are not contained in regular
           // menu trees, and have only been loaded for the additional
           // translation that happens here, so as to be able to display them in
-          // the breadcumb for the current page.
+          // the breadcrumb for the current page.
           // @see _menu_tree_check_access()
           // @see _menu_link_translate()
           if (strpos($link['href'], '%') !== FALSE) {
@@ -2413,7 +2434,8 @@ function menu_set_active_trail($new_trail = NULL) {
         }
         $tree = $curr['below'] ? $curr['below'] : array();
       }
-      list($key, $curr) = each($tree);
+      $curr = current($tree);
+      next($tree);
     }
     // Make sure the current page is in the trail to build the page title, by
     // appending either the preferred link or the menu router item for the
@@ -2430,7 +2452,7 @@ function menu_set_active_trail($new_trail = NULL) {
  * Looks up the preferred menu link for a given system path.
  *
  * @param $path
- *   The path, for example 'node/5'. The function will find the corresponding
+ *   The path; for example, 'node/5'. The function will find the corresponding
  *   menu link ('node/5' if it exists, or fallback to 'node/%').
  * @param $selected_menu
  *   The name of a menu used to restrict the search for a preferred menu link.
@@ -2461,6 +2483,9 @@ function menu_link_get_preferred($path = NULL, $selected_menu = NULL) {
     // untranslated paths). Afterwards, the most relevant path is picked from
     // the menus, ordered by menu preference.
     $item = menu_get_item($path);
+    if ($item === FALSE) {
+      return FALSE;
+    }
     $path_candidates = array();
     // 1. The current item href.
     $path_candidates[$item['href']] = $item['href'];
@@ -2487,6 +2512,7 @@ function menu_link_get_preferred($path = NULL, $selected_menu = NULL) {
     $query->addField('ml', 'weight', 'link_weight');
     $query->fields('m');
     $query->condition('ml.link_path', $path_candidates, 'IN');
+    $query->addTag('preferred_menu_links');
 
     // Sort candidates by link path and menu name.
     $candidates = array();
@@ -2602,10 +2628,30 @@ function menu_get_active_breadcrumb() {
  */
 function menu_get_active_title() {
   $active_trail = menu_get_active_trail();
+  $local_task_title = NULL;
 
   foreach (array_reverse($active_trail) as $item) {
-    if (!(bool) ($item['type'] & MENU_IS_LOCAL_TASK)) {
-      return $item['title'];
+    // Local task titles are displayed as tabs and therefore should not be
+    // repeated as the page title. However, if the local task appears in a
+    // top-level menu, it is no longer a "local task" anymore (the front page
+    // of the site does not have tabs) so it is better to use the local task
+    // title in that case than to fall back on the front page link in the
+    // active trail (which is usually "Home" and would not make sense in this
+    // context).
+    if ((bool) ($item['type'] & MENU_IS_LOCAL_TASK)) {
+      // A local task title is being skipped; track it in case it needs to be
+      // used later.
+      $local_task_title = $item['title'];
+    }
+    else {
+      // This is not a local task, so use it for the page title (unless the
+      // conditions described above are met).
+      if (isset($local_task_title) && isset($item['href']) && $item['href'] == '<front>') {
+        return $local_task_title;
+      }
+      else {
+        return $item['title'];
+      }
     }
   }
 }
@@ -2643,7 +2689,7 @@ function menu_link_load($mlid) {
 }
 
 /**
- * Clears the cached cached data for a single named menu.
+ * Clears the cached data for a single named menu.
  */
 function menu_cache_clear($menu_name = 'navigation') {
   $cache_cleared = &drupal_static(__FUNCTION__, array());
@@ -2684,6 +2730,21 @@ function menu_reset_static_cache() {
   drupal_static_reset('menu_link_get_preferred');
 }
 
+/**
+ * Checks whether a menu_rebuild() is necessary.
+ */
+function _menu_check_rebuild() {
+  // To absolutely ensure that the menu rebuild is required, re-load the
+  // variables in case they were set by another process.
+  $variables = variable_initialize();
+  if (empty($variables['menu_rebuild_needed']) && !empty($variables['menu_masks'])) {
+    unset($GLOBALS['conf']['menu_rebuild_needed']);
+    $GLOBALS['conf']['menu_masks'] = $variables['menu_masks'];
+    return FALSE;
+  }
+  return TRUE;
+}
+
 /**
  * Populates the database tables used by various menu functions.
  *
@@ -2704,6 +2765,14 @@ function menu_rebuild() {
     // We choose to block here since otherwise the router item may not
     // be available in menu_execute_active_handler() resulting in a 404.
     lock_wait('menu_rebuild');
+
+    if (_menu_check_rebuild()) {
+      // If we get here and menu_masks was not set, then it is possible a menu
+      // is being reloaded, or that the process rebuilding the menu was unable
+      // to complete successfully. A missing menu_masks variable could result
+      // in a 404, so re-run the function.
+      return menu_rebuild();
+    }
     return FALSE;
   }
 
@@ -2728,6 +2797,12 @@ function menu_rebuild() {
     $transaction->rollback();
     watchdog_exception('menu', $e);
   }
+  // Explicitly commit the transaction now; this ensures that the database
+  // operations during the menu rebuild are committed before the lock is made
+  // available again, since locks may not always reside in the same database
+  // connection. The lock is acquired outside of the transaction so should also
+  // be released outside of it.
+  unset($transaction);
 
   lock_release('menu_rebuild');
   return TRUE;

includes/module.inc

@@ -227,6 +227,10 @@ function system_list_reset() {
   drupal_static_reset('list_themes');
   cache_clear_all('bootstrap_modules', 'cache_bootstrap');
   cache_clear_all('system_list', 'cache_bootstrap');
+
+  // Clean up the bootstrap file scan cache.
+  drupal_static_reset('_drupal_file_scan_cache');
+  cache_clear_all('_drupal_file_scan_cache', 'cache_bootstrap');
 }
 
 /**
@@ -265,11 +269,11 @@ function _module_build_dependencies($files) {
 /**
  * Determines whether a given module exists.
  *
- * @param $module
+ * @param string $module
  *   The name of the module (without the .module extension).
  *
- * @return
- *   TRUE if the module is both installed and enabled.
+ * @return bool
+ *   TRUE if the module is both installed and enabled, FALSE otherwise.
  */
 function module_exists($module) {
   $list = module_list();
@@ -320,16 +324,27 @@ function module_load_install($module) {
  *   The name of the included file, if successful; FALSE otherwise.
  */
 function module_load_include($type, $module, $name = NULL) {
+  static $files = array();
+
   if (!isset($name)) {
     $name = $module;
   }
 
+  $key = $type . ':' . $module . ':' . $name;
+  if (isset($files[$key])) {
+    return $files[$key];
+  }
+
   if (function_exists('drupal_get_path')) {
     $file = DRUPAL_ROOT . '/' . drupal_get_path('module', $module) . "/$name.$type";
     if (is_file($file)) {
       require_once $file;
+      $files[$key] = $file;
       return $file;
     }
+    else {
+      $files[$key] = FALSE;
+    }
   }
   return FALSE;
 }
@@ -365,20 +380,22 @@ function module_load_all_includes($type, $name = NULL) {
  * - Invoke hook_modules_installed().
  * - Invoke hook_modules_enabled().
  *
- * @param $module_list
+ * @param string[] $module_list
  *   An array of module names.
- * @param $enable_dependencies
+ * @param bool $enable_dependencies
  *   If TRUE, dependencies will automatically be added and enabled in the
  *   correct order. This incurs a significant performance cost, so use FALSE
  *   if you know $module_list is already complete and in the correct order.
  *
- * @return
+ * @return bool
  *   FALSE if one or more dependencies are missing, TRUE otherwise.
  *
  * @see hook_install()
  * @see hook_enable()
  * @see hook_modules_installed()
  * @see hook_modules_enabled()
+ * @see module_disable()
+ * @see drupal_uninstall_modules()
  */
 function module_enable($module_list, $enable_dependencies = TRUE) {
   if ($enable_dependencies) {
@@ -387,7 +404,11 @@ function module_enable($module_list, $enable_dependencies = TRUE) {
     // Create an associative array with weights as values.
     $module_list = array_flip(array_values($module_list));
 
-    while (list($module) = each($module_list)) {
+    // The array is iterated over manually (instead of using a foreach) because
+    // modules may be added to the list within the loop and we need to process
+    // them.
+    while ($module = key($module_list)) {
+      next($module_list);
       if (!isset($module_data[$module])) {
         // This module is not found in the filesystem, abort.
         return FALSE;
@@ -505,12 +526,15 @@ function module_enable($module_list, $enable_dependencies = TRUE) {
 /**
  * Disables a given set of modules.
  *
- * @param $module_list
+ * @param string[] $module_list
  *   An array of module names.
- * @param $disable_dependents
+ * @param bool $disable_dependents
  *   If TRUE, dependent modules will automatically be added and disabled in the
  *   correct order. This incurs a significant performance cost, so use FALSE
  *   if you know $module_list is already complete and in the correct order.
+ *
+ * @see drupal_uninstall_modules()
+ * @see module_enable()
  */
 function module_disable($module_list, $disable_dependents = TRUE) {
   if ($disable_dependents) {
@@ -520,7 +544,11 @@ function module_disable($module_list, $disable_dependents = TRUE) {
     $module_list = array_flip(array_values($module_list));
 
     $profile = drupal_get_profile();
-    while (list($module) = each($module_list)) {
+    // The array is iterated over manually (instead of using a foreach) because
+    // modules may be added to the list within the loop and we need to process
+    // them.
+    while ($module = key($module_list)) {
+      next($module_list);
       if (!isset($module_data[$module]) || !$module_data[$module]->status) {
         // This module doesn't exist or is already disabled, skip it.
         unset($module_list[$module]);
@@ -610,9 +638,40 @@ function module_disable($module_list, $disable_dependents = TRUE) {
  * just models that you can modify. Only the hooks implemented within modules
  * are executed when running Drupal.
  *
- * See also @link themeable the themeable group page. @endlink
+ * @see themeable
+ * @see callbacks
  */
 
+ /**
+  * @defgroup callbacks Callbacks
+  * @{
+  * Callback function signatures.
+  *
+  * Drupal's API sometimes uses callback functions to allow you to define how
+  * some type of processing happens. A callback is a function with a defined
+  * signature, which you define in a module. Then you pass the function name as
+  * a parameter to a Drupal API function or return it as part of a hook
+  * implementation return value, and your function is called at an appropriate
+  * time. For instance, when setting up batch processing you might need to
+  * provide a callback function for each processing step and/or a callback for
+  * when processing is finished; you would do that by defining these functions
+  * and passing their names into the batch setup function.
+  *
+  * Callback function signatures, like hook definitions, are described by
+  * creating and documenting dummy functions in a *.api.php file; normally, the
+  * dummy callback function's name should start with "callback_", and you should
+  * document the parameters and return value and provide a sample function body.
+  * Then your API documentation can refer to this callback function in its
+  * documentation. A user of your API can usually name their callback function
+  * anything they want, although a standard name would be to replace "callback_"
+  * with the module name.
+  *
+  * @see hooks
+  * @see themeable
+  *
+  * @}
+  */
+
 /**
  * Determines whether a module implements a hook.
  *
@@ -645,12 +704,16 @@ function module_hook($module, $hook) {
 /**
  * Determines which modules are implementing a hook.
  *
- * @param $hook
+ * Lazy-loaded include files specified with "group" via hook_hook_info() or
+ * hook_module_implements_alter() will be automatically included by this
+ * function when necessary.
+ *
+ * @param string $hook
  *   The name of the hook (e.g. "help" or "menu").
- * @param $sort
+ * @param bool $sort
  *   By default, modules are ordered by weight and filename, settings this option
  *   to TRUE, module list will be ordered by module name.
- * @param $reset
+ * @param bool $reset
  *   For internal use only: Whether to force the stored list of hook
  *   implementations to be regenerated (such as after enabling a new module,
  *   before processing hook_enable).
@@ -665,8 +728,10 @@ function module_implements($hook, $sort = FALSE, $reset = FALSE) {
   static $drupal_static_fast;
   if (!isset($drupal_static_fast)) {
     $drupal_static_fast['implementations'] = &drupal_static(__FUNCTION__);
+    $drupal_static_fast['verified'] = &drupal_static(__FUNCTION__ . ':verified');
   }
   $implementations = &$drupal_static_fast['implementations'];
+  $verified = &$drupal_static_fast['verified'];
 
   // We maintain a persistent cache of hook implementations in addition to the
   // static cache to avoid looping through every module and every hook on each
@@ -680,14 +745,19 @@ function module_implements($hook, $sort = FALSE, $reset = FALSE) {
   // per request.
   if ($reset) {
     $implementations = array();
+    $verified = array();
     cache_set('module_implements', array(), 'cache_bootstrap');
     drupal_static_reset('module_hook_info');
     drupal_static_reset('drupal_alter');
     cache_clear_all('hook_info', 'cache_bootstrap');
+    cache_clear_all('system_cache_tables', 'cache');
     return;
   }
 
   // Fetch implementations from cache.
+  // This happens on the first call to module_implements(*, *, FALSE) during a
+  // request, but also when $implementations have been reset, e.g. after
+  // module_enable().
   if (empty($implementations)) {
     $implementations = cache_get('module_implements', 'cache_bootstrap');
     if ($implementations === FALSE) {
@@ -696,12 +766,17 @@ function module_implements($hook, $sort = FALSE, $reset = FALSE) {
     else {
       $implementations = $implementations->data;
     }
+    // Forget all previously "verified" hooks, in case that $implementations
+    // were cleared via drupal_static_reset('module_implements') instead of
+    // module_implements(*, *, TRUE).
+    $verified = array();
   }
 
   if (!isset($implementations[$hook])) {
     // The hook is not cached, so ensure that whether or not it has
     // implementations, that the cache is updated at the end of the request.
     $implementations['#write_cache'] = TRUE;
+    // Discover implementations for this hook.
     $hook_info = module_hook_info();
     $implementations[$hook] = array();
     $list = module_list(FALSE, FALSE, $sort);
@@ -713,13 +788,31 @@ function module_implements($hook, $sort = FALSE, $reset = FALSE) {
         $implementations[$hook][$module] = $include_file ? $hook_info[$hook]['group'] : FALSE;
       }
     }
-    // Allow modules to change the weight of specific implementations but avoid
+    // Allow modules to change the weight of specific implementations, but avoid
     // an infinite loop.
     if ($hook != 'module_implements_alter') {
+      // Remember the implementations before hook_module_implements_alter().
+      $implementations_before = $implementations[$hook];
       drupal_alter('module_implements', $implementations[$hook], $hook);
+      // Verify implementations that were added or modified.
+      foreach (array_diff_assoc($implementations[$hook], $implementations_before) as $module => $group) {
+        // If drupal_alter('module_implements') changed or added a $group, the
+        // respective file needs to be included.
+        if ($group) {
+          module_load_include('inc', $module, "$module.$group");
+        }
+        // If a new implementation was added, verify that the function exists.
+        if (!function_exists($module . '_' . $hook)) {
+          unset($implementations[$hook][$module]);
+        }
+      }
     }
+    // Implementations for this hook are now "verified".
+    $verified[$hook] = TRUE;
   }
-  else {
+  elseif (!isset($verified[$hook])) {
+    // Implementations for this hook were in the cache, but they are not
+    // "verified" yet.
     foreach ($implementations[$hook] as $module => $group) {
       // If this hook implementation is stored in a lazy-loaded file, so include
       // that file first.
@@ -738,6 +831,7 @@ function module_implements($hook, $sort = FALSE, $reset = FALSE) {
         $implementations['#write_cache'] = TRUE;
       }
     }
+    $verified[$hook] = TRUE;
   }
 
   return array_keys($implementations[$hook]);
@@ -802,11 +896,13 @@ function module_hook_info() {
  * @see module_implements()
  */
 function module_implements_write_cache() {
+  // The list of implementations includes vital modules only before full
+  // bootstrap, so do not write cache if we are not fully bootstrapped yet.
+  if (drupal_get_bootstrap_phase() != DRUPAL_BOOTSTRAP_FULL) {
+    return;
+  }
   $implementations = &drupal_static('module_implements');
-  // Check whether we need to write the cache. We do not want to cache hooks
-  // which are only invoked on HTTP POST requests since these do not need to be
-  // optimized as tightly, and not doing so keeps the cache entry smaller.
-  if (isset($implementations['#write_cache']) && ($_SERVER['REQUEST_METHOD'] == 'GET' || $_SERVER['REQUEST_METHOD'] == 'HEAD')) {
+  if (isset($implementations['#write_cache'])) {
     unset($implementations['#write_cache']);
     cache_set('module_implements', $implementations, 'cache_bootstrap');
   }
@@ -815,6 +911,9 @@ function module_implements_write_cache() {
 /**
  * Invokes a hook in a particular module.
  *
+ * All arguments are passed by value. Use drupal_alter() if you need to pass
+ * arguments by reference.
+ *
  * @param $module
  *   The name of the module (without the .module extension).
  * @param $hook
@@ -824,6 +923,8 @@ function module_implements_write_cache() {
  *
  * @return
  *   The return value of the hook implementation.
+ *
+ * @see drupal_alter()
  */
 function module_invoke($module, $hook) {
   $args = func_get_args();
@@ -837,6 +938,9 @@ function module_invoke($module, $hook) {
 /**
  * Invokes a hook in all enabled modules that implement it.
  *
+ * All arguments are passed by value. Use drupal_alter() if you need to pass
+ * arguments by reference.
+ *
  * @param $hook
  *   The name of the hook to invoke.
  * @param ...
@@ -844,7 +948,11 @@ function module_invoke($module, $hook) {
  *
  * @return
  *   An array of return values of the hook implementations. If modules return
- *   arrays from their implementations, those are merged into one array.
+ *   arrays from their implementations, those are merged into one array
+ *   recursively. Note: integer keys in arrays will be lost, as the merge is
+ *   done using array_merge_recursive().
+ *
+ * @see drupal_alter()
  */
 function module_invoke_all($hook) {
   $args = func_get_args();

includes/pager.inc

@@ -321,9 +321,19 @@ function theme_pager($variables) {
   $tags = $variables['tags'];
   $element = $variables['element'];
   $parameters = $variables['parameters'];
-  $quantity = $variables['quantity'];
+  $quantity = empty($variables['quantity']) ? 0 : $variables['quantity'];
   global $pager_page_array, $pager_total;
 
+  // Nothing to do if there is no pager.
+  if (!isset($pager_page_array[$element]) || !isset($pager_total[$element])) {
+    return;
+  }
+
+  // Nothing to do if there is only one page.
+  if ($pager_total[$element] <= 1) {
+    return;
+  }
+
   // Calculate various markers within this pager piece:
   // Middle is used to "center" pages around the current page.
   $pager_middle = ceil($quantity / 2);
@@ -455,6 +465,11 @@ function theme_pager_first($variables) {
   global $pager_page_array;
   $output = '';
 
+  // Nothing to do if there is no pager.
+  if (!isset($pager_page_array[$element])) {
+    return;
+  }
+
   // If we are anywhere but the first page
   if ($pager_page_array[$element] > 0) {
     $output = theme('pager_link', array('text' => $text, 'page_new' => pager_load_array(0, $element, $pager_page_array), 'element' => $element, 'parameters' => $parameters));
@@ -485,6 +500,11 @@ function theme_pager_previous($variables) {
   global $pager_page_array;
   $output = '';
 
+  // Nothing to do if there is no pager.
+  if (!isset($pager_page_array[$element])) {
+    return;
+  }
+
   // If we are anywhere but the first page
   if ($pager_page_array[$element] > 0) {
     $page_new = pager_load_array($pager_page_array[$element] - $interval, $element, $pager_page_array);
@@ -524,6 +544,11 @@ function theme_pager_next($variables) {
   global $pager_page_array, $pager_total;
   $output = '';
 
+  // Nothing to do if there is no pager.
+  if (!isset($pager_page_array[$element]) || !isset($pager_total[$element])) {
+    return;
+  }
+
   // If we are anywhere but the last page
   if ($pager_page_array[$element] < ($pager_total[$element] - 1)) {
     $page_new = pager_load_array($pager_page_array[$element] + $interval, $element, $pager_page_array);
@@ -560,6 +585,11 @@ function theme_pager_last($variables) {
   global $pager_page_array, $pager_total;
   $output = '';
 
+  // Nothing to do if there is no pager.
+  if (!isset($pager_page_array[$element]) || !isset($pager_total[$element])) {
+    return;
+  }
+
   // If we are anywhere but the last page
   if ($pager_page_array[$element] < ($pager_total[$element] - 1)) {
     $output = theme('pager_link', array('text' => $text, 'page_new' => pager_load_array($pager_total[$element] - 1, $element, $pager_page_array), 'element' => $element, 'parameters' => $parameters));

includes/password.inc

@@ -140,7 +140,7 @@ function _password_enforce_log2_boundaries($count_log2) {
  * @param $algo
  *   The string name of a hashing algorithm usable by hash(), like 'sha256'.
  * @param $password
- *   The plain-text password to hash.
+ *   Plain-text password up to 512 bytes (128 to 512 UTF-8 characters) to hash.
  * @param $setting
  *   An existing hash or the output of _password_generate_salt().  Must be
  *   at least 12 characters (the settings and salt).
@@ -150,6 +150,10 @@ function _password_enforce_log2_boundaries($count_log2) {
  *   The return string will be truncated at DRUPAL_HASH_LENGTH characters max.
  */
 function _password_crypt($algo, $password, $setting) {
+  // Prevent DoS attacks by refusing to hash large passwords.
+  if (strlen($password) > 512) {
+    return FALSE;
+  }
   // The first 12 characters of an existing hash are its setting string.
   $setting = substr($setting, 0, 12);
 

includes/path.inc

@@ -347,7 +347,8 @@ function drupal_match_path($path, $patterns) {
  * drupal_bootstrap(DRUPAL_BOOTSTRAP_FULL) makes this function available.
  *
  * @return
- *   The current Drupal URL path.
+ *   The current Drupal URL path. The path is untrusted user input and must be
+ *   treated as such.
  *
  * @see request_path()
  */
@@ -465,13 +466,15 @@ function path_delete($criteria) {
     $criteria = array('pid' => $criteria);
   }
   $path = path_load($criteria);
-  $query = db_delete('url_alias');
-  foreach ($criteria as $field => $value) {
-    $query->condition($field, $value);
+  if (isset($path['source'])) {
+    $query = db_delete('url_alias');
+    foreach ($criteria as $field => $value) {
+      $query->condition($field, $value);
+    }
+    $query->execute();
+    module_invoke_all('path_delete', $path);
+    drupal_clear_path_cache($path['source']);
   }
-  $query->execute();
-  module_invoke_all('path_delete', $path);
-  drupal_clear_path_cache($path['source']);
 }
 
 /**
@@ -560,8 +563,8 @@ function drupal_valid_path($path, $dynamic_allowed = FALSE) {
   elseif ($dynamic_allowed && preg_match('/\/\%/', $path)) {
     // Path is dynamic (ie 'user/%'), so check directly against menu_router table.
     if ($item = db_query("SELECT * FROM {menu_router} where path = :path", array(':path' => $path))->fetchAssoc()) {
-      $item['link_path']  = $form_item['link_path'];
-      $item['link_title'] = $form_item['link_title'];
+      $item['link_path']  = $item['path'];
+      $item['link_title'] = $item['title'];
       $item['external']   = FALSE;
       $item['options'] = '';
       _menu_link_translate($item);

includes/registry.inc

@@ -10,7 +10,7 @@
  * @{
  * The code registry engine.
  *
- * Drupal maintains an internal registry of all functions or classes in the
+ * Drupal maintains an internal registry of all interfaces or classes in the
  * system, allowing it to lazy-load code files as needed (reducing the amount
  * of code that must be parsed on each request).
  */
@@ -19,7 +19,6 @@
  * Does the work for registry_update().
  */
 function _registry_update() {
-
   // The registry serves as a central autoloader for all classes, including
   // the database query builders. However, the registry rebuild process
   // requires write ability to the database, which means having access to the
@@ -33,6 +32,11 @@ function _registry_update() {
   require_once DRUPAL_ROOT . '/includes/database/select.inc';
   require_once DRUPAL_ROOT . '/includes/database/' . $driver . '/query.inc';
 
+  // During the first registry rebuild in a request, we check all the files.
+  // During subsequent rebuilds, we only add new files. It makes the rebuilding
+  // process faster during installation of modules.
+  static $check_existing_files = TRUE;
+
   // Get current list of modules and their files.
   $modules = db_query("SELECT * FROM {system} WHERE type = 'module'")->fetchAll();
   // Get the list of files we are going to parse.
@@ -55,6 +59,9 @@ function _registry_update() {
     $files["$filename"] = array('module' => '', 'weight' => 0);
   }
 
+  // Initialize an empty array for the unchanged files.
+  $unchanged_files = array();
+
   $transaction = db_transaction();
   try {
     // Allow modules to manually modify the list of files before the registry
@@ -63,10 +70,19 @@ function _registry_update() {
     // list can then be added to the list of files that the registry will parse,
     // or modify attributes of a file.
     drupal_alter('registry_files', $files, $modules);
+
     foreach (registry_get_parsed_files() as $filename => $file) {
       // Add the hash for those files we have already parsed.
       if (isset($files[$filename])) {
-        $files[$filename]['hash'] = $file['hash'];
+        if ($check_existing_files === TRUE) {
+          $files[$filename]['hash'] = $file['hash'];
+        }
+        else {
+          // Ignore that file for this request, it has been parsed previously
+          // and it is unlikely it has changed.
+          unset($files[$filename]);
+          $unchanged_files[$filename] = $file;
+        }
       }
       else {
         // Flush the registry of resources in files that are no longer on disc
@@ -79,8 +95,12 @@ function _registry_update() {
           ->execute();
       }
     }
+
     $parsed_files = _registry_parse_files($files);
 
+    // Add unchanged files to the files.
+    $files += $unchanged_files;
+
     $unchanged_resources = array();
     $lookup_cache = array();
     if ($cache = cache_get('lookup_cache', 'cache_bootstrap')) {
@@ -89,12 +109,10 @@ function _registry_update() {
     foreach ($lookup_cache as $key => $file) {
       // If the file for this cached resource is carried over unchanged from
       // the last registry build, then we can safely re-cache it.
-      if ($file && in_array($file, array_keys($files)) && !in_array($file, $parsed_files)) {
+      if ($file && isset($files[$file]) && !in_array($file, $parsed_files, TRUE)) {
         $unchanged_resources[$key] = $file;
       }
     }
-    module_implements('', FALSE, TRUE);
-    _registry_check_code(REGISTRY_RESET_LOOKUP_CACHE);
   }
   catch (Exception $e) {
     $transaction->rollback();
@@ -102,6 +120,13 @@ function _registry_update() {
     throw $e;
   }
 
+  module_implements('', FALSE, TRUE);
+  _registry_check_code(REGISTRY_RESET_LOOKUP_CACHE);
+
+  // During the next run in this request, don't bother re-checking existing
+  // files.
+  $check_existing_files = FALSE;
+
   // We have some unchanged resources, warm up the cache - no need to pay
   // for looking them up again.
   if (count($unchanged_resources) > 0) {
@@ -120,7 +145,10 @@ function registry_get_parsed_files() {
 }
 
 /**
- * Parse all files that have changed since the registry was last built, and save their function and class listings.
+ * Parse all changed files and save their interface and class listings.
+ *
+ * Parse all files that have changed since the registry was last built, and save
+ * their interface and class listings.
  *
  * @param $files
  *  The list of files to check and parse.
@@ -149,7 +177,7 @@ function _registry_parse_files($files) {
 }
 
 /**
- * Parse a file and save its function and class listings.
+ * Parse a file and save its interface and class listings.
  *
  * @param $filename
  *   Name of the file we are going to parse.
@@ -161,7 +189,7 @@ function _registry_parse_files($files) {
  *   (optional) Weight of the module.
  */
 function _registry_parse_file($filename, $contents, $module = '', $weight = 0) {
-  if (preg_match_all('/^\s*(?:abstract|final)?\s*(class|interface)\s+([a-zA-Z0-9_]+)/m', $contents, $matches)) {
+  if (preg_match_all('/^\s*(?:abstract|final)?\s*(class|interface|trait)\s+([a-zA-Z0-9_]+)/m', $contents, $matches)) {
     foreach ($matches[2] as $key => $name) {
       db_merge('registry')
         ->key(array(

includes/request-sanitizer.inc

@@ -0,0 +1,114 @@
+<?php
+
+/**
+ * @file
+ * Contains code for sanitizing user input from the request.
+ */
+
+/**
+ * Sanitizes user input from the request.
+ */
+class DrupalRequestSanitizer {
+
+  /**
+   * Tracks whether the request was already sanitized.
+   */
+  protected static $sanitized = FALSE;
+
+  /**
+   * Modifies the request to strip dangerous keys from user input.
+   */
+  public static function sanitize() {
+    if (!self::$sanitized) {
+      $whitelist = variable_get('sanitize_input_whitelist', array());
+      $log_sanitized_keys = variable_get('sanitize_input_logging', FALSE);
+
+      // Process query string parameters.
+      $get_sanitized_keys = array();
+      $_GET = self::stripDangerousValues($_GET, $whitelist, $get_sanitized_keys);
+      if ($log_sanitized_keys && $get_sanitized_keys) {
+        _drupal_trigger_error_with_delayed_logging(format_string('Potentially unsafe keys removed from query string parameters (GET): @keys', array('@keys' => implode(', ', $get_sanitized_keys))), E_USER_NOTICE);
+      }
+
+      // Process request body parameters.
+      $post_sanitized_keys = array();
+      $_POST = self::stripDangerousValues($_POST, $whitelist, $post_sanitized_keys);
+      if ($log_sanitized_keys && $post_sanitized_keys) {
+        _drupal_trigger_error_with_delayed_logging(format_string('Potentially unsafe keys removed from request body parameters (POST): @keys', array('@keys' => implode(', ', $post_sanitized_keys))), E_USER_NOTICE);
+      }
+
+      // Process cookie parameters.
+      $cookie_sanitized_keys = array();
+      $_COOKIE = self::stripDangerousValues($_COOKIE, $whitelist, $cookie_sanitized_keys);
+      if ($log_sanitized_keys && $cookie_sanitized_keys) {
+        _drupal_trigger_error_with_delayed_logging(format_string('Potentially unsafe keys removed from cookie parameters (COOKIE): @keys', array('@keys' => implode(', ', $cookie_sanitized_keys))), E_USER_NOTICE);
+      }
+
+      $request_sanitized_keys = array();
+      $_REQUEST = self::stripDangerousValues($_REQUEST, $whitelist, $request_sanitized_keys);
+
+      self::$sanitized = TRUE;
+    }
+  }
+
+  /**
+   * Removes the destination if it is dangerous.
+   *
+   * Note this can only be called after common.inc has been included.
+   *
+   * @return bool
+   *   TRUE if the destination has been removed from $_GET, FALSE if not.
+   */
+  public static function cleanDestination() {
+    $dangerous_keys = array();
+    $log_sanitized_keys = variable_get('sanitize_input_logging', FALSE);
+
+    $parts = drupal_parse_url($_GET['destination']);
+    // If there is a query string, check its query parameters.
+    if (!empty($parts['query'])) {
+      $whitelist = variable_get('sanitize_input_whitelist', array());
+
+      self::stripDangerousValues($parts['query'], $whitelist, $dangerous_keys);
+      if (!empty($dangerous_keys)) {
+        // The destination is removed rather than sanitized to mirror the
+        // handling of external destinations.
+        unset($_GET['destination']);
+        unset($_REQUEST['destination']);
+        if ($log_sanitized_keys) {
+          trigger_error(format_string('Potentially unsafe destination removed from query string parameters (GET) because it contained the following keys: @keys', array('@keys' => implode(', ', $dangerous_keys))));
+        }
+        return TRUE;
+      }
+    }
+    return FALSE;
+  }
+
+  /**
+   * Strips dangerous keys from the provided input.
+   *
+   * @param mixed $input
+   *   The input to sanitize.
+   * @param string[] $whitelist
+   *   An array of keys to whitelist as safe.
+   * @param string[] $sanitized_keys
+   *   An array of keys that have been removed.
+   *
+   * @return mixed
+   *   The sanitized input.
+   */
+  protected static function stripDangerousValues($input, array $whitelist, array &$sanitized_keys) {
+    if (is_array($input)) {
+      foreach ($input as $key => $value) {
+        if ($key !== '' && is_string($key) && $key[0] === '#' && !in_array($key, $whitelist, TRUE)) {
+          unset($input[$key]);
+          $sanitized_keys[] = $key;
+        }
+        else {
+          $input[$key] = self::stripDangerousValues($input[$key], $whitelist, $sanitized_keys);
+        }
+      }
+    }
+    return $input;
+  }
+
+}

includes/session.inc

@@ -79,7 +79,7 @@ function _drupal_session_read($sid) {
   // Handle the case of first time visitors and clients that don't store
   // cookies (eg. web crawlers).
   $insecure_session_name = substr(session_name(), 1);
-  if (!isset($_COOKIE[session_name()]) && !isset($_COOKIE[$insecure_session_name])) {
+  if (empty($sid) || (!isset($_COOKIE[session_name()]) && !isset($_COOKIE[$insecure_session_name]))) {
     $user = drupal_anonymous_user();
     return '';
   }
@@ -163,7 +163,7 @@ function _drupal_session_write($sid, $value) {
   try {
     if (!drupal_save_session()) {
       // We don't have anything to do if we are not allowed to save the session.
-      return;
+      return TRUE;
     }
 
     // Check whether $_SESSION has been changed in this request.
@@ -263,10 +263,10 @@ function drupal_session_initialize() {
     // Less random sessions (which are much faster to generate) are used for
     // anonymous users than are generated in drupal_session_regenerate() when
     // a user becomes authenticated.
-    session_id(drupal_hash_base64(uniqid(mt_rand(), TRUE)));
+    session_id(drupal_random_key());
     if ($is_https && variable_get('https', FALSE)) {
       $insecure_session_name = substr(session_name(), 1);
-      $session_id = drupal_hash_base64(uniqid(mt_rand(), TRUE));
+      $session_id = drupal_random_key();
       $_COOKIE[$insecure_session_name] = $session_id;
     }
   }
@@ -360,7 +360,7 @@ function drupal_session_regenerate() {
       $old_insecure_session_id = $_COOKIE[$insecure_session_name];
     }
     $params = session_get_cookie_params();
-    $session_id = drupal_hash_base64(uniqid(mt_rand(), TRUE) . drupal_random_bytes(55));
+    $session_id = drupal_random_key();
     // If a session cookie lifetime is set, the session will expire
     // $params['lifetime'] seconds from the current request. If it is not set,
     // it will expire when the browser is closed.
@@ -371,8 +371,11 @@ function drupal_session_regenerate() {
 
   if (drupal_session_started()) {
     $old_session_id = session_id();
+    _drupal_session_regenerate_existing();
+  }
+  else {
+    session_id(drupal_random_key());
   }
-  session_id(drupal_hash_base64(uniqid(mt_rand(), TRUE) . drupal_random_bytes(55)));
 
   if (isset($old_session_id)) {
     $params = session_get_cookie_params();
@@ -412,6 +415,26 @@ function drupal_session_regenerate() {
   date_default_timezone_set(drupal_get_user_timezone());
 }
 
+/**
+ * Regenerates an existing session.
+ */
+function _drupal_session_regenerate_existing() {
+  global $user;
+  // Preserve existing settings for the saving of sessions.
+  $original_save_session_status = drupal_save_session();
+  // Turn off saving of sessions.
+  drupal_save_session(FALSE);
+  session_write_close();
+  drupal_session_started(FALSE);
+  // Preserve the user object, as starting a new session will reset it.
+  $original_user = $user;
+  session_id(drupal_random_key());
+  drupal_session_start();
+  $user = $original_user;
+  // Restore the original settings for the saving of sessions.
+  drupal_save_session($original_save_session_status);
+}
+
 /**
  * Session handler assigned by session_set_save_handler().
  *
@@ -425,7 +448,7 @@ function _drupal_session_destroy($sid) {
 
   // Nothing to do if we are not allowed to change the session.
   if (!drupal_save_session()) {
-    return;
+    return TRUE;
   }
 
   // Delete session data.
@@ -446,6 +469,8 @@ function _drupal_session_destroy($sid) {
   elseif (variable_get('https', FALSE)) {
     _drupal_session_delete_cookie('S' . session_name(), TRUE);
   }
+
+  return TRUE;
 }
 
 /**

includes/stream_wrappers.inc

@@ -93,7 +93,7 @@ define('STREAM_WRAPPERS_LOCAL_NORMAL', STREAM_WRAPPERS_LOCAL | STREAM_WRAPPERS_N
 /**
  * Generic PHP stream wrapper interface.
  *
- * @see http://www.php.net/manual/en/class.streamwrapper.php
+ * @see http://www.php.net/manual/class.streamwrapper.php
  */
 interface StreamWrapperInterface {
   public function stream_open($uri, $mode, $options, &$opened_url);
@@ -133,7 +133,7 @@ interface DrupalStreamWrapperInterface extends StreamWrapperInterface {
    * @param $uri
    *   A string containing the URI that should be used for this instance.
    */
-  function setUri($uri);
+  public function setUri($uri);
 
   /**
    * Returns the stream resource URI.
@@ -219,7 +219,6 @@ interface DrupalStreamWrapperInterface extends StreamWrapperInterface {
   public function dirname($uri = NULL);
 }
 
-
 /**
  * Drupal stream wrapper base class for local files.
  *
@@ -401,7 +400,7 @@ abstract class DrupalLocalStreamWrapper implements DrupalStreamWrapperInterface
    * @return
    *   Returns TRUE if file was opened successfully.
    *
-   * @see http://php.net/manual/en/streamwrapper.stream-open.php
+   * @see http://php.net/manual/streamwrapper.stream-open.php
    */
   public function stream_open($uri, $mode, $options, &$opened_path) {
     $this->uri = $uri;
@@ -429,7 +428,7 @@ abstract class DrupalLocalStreamWrapper implements DrupalStreamWrapperInterface
    * @return
    *   Always returns TRUE at the present time.
    *
-   * @see http://php.net/manual/en/streamwrapper.stream-lock.php
+   * @see http://php.net/manual/streamwrapper.stream-lock.php
    */
   public function stream_lock($operation) {
     if (in_array($operation, array(LOCK_SH, LOCK_EX, LOCK_UN, LOCK_NB))) {
@@ -448,7 +447,7 @@ abstract class DrupalLocalStreamWrapper implements DrupalStreamWrapperInterface
    * @return
    *   The string that was read, or FALSE in case of an error.
    *
-   * @see http://php.net/manual/en/streamwrapper.stream-read.php
+   * @see http://php.net/manual/streamwrapper.stream-read.php
    */
   public function stream_read($count) {
     return fread($this->handle, $count);
@@ -463,7 +462,7 @@ abstract class DrupalLocalStreamWrapper implements DrupalStreamWrapperInterface
    * @return
    *   The number of bytes written (integer).
    *
-   * @see http://php.net/manual/en/streamwrapper.stream-write.php
+   * @see http://php.net/manual/streamwrapper.stream-write.php
    */
   public function stream_write($data) {
     return fwrite($this->handle, $data);
@@ -475,7 +474,7 @@ abstract class DrupalLocalStreamWrapper implements DrupalStreamWrapperInterface
    * @return
    *   TRUE if end-of-file has been reached.
    *
-   * @see http://php.net/manual/en/streamwrapper.stream-eof.php
+   * @see http://php.net/manual/streamwrapper.stream-eof.php
    */
   public function stream_eof() {
     return feof($this->handle);
@@ -492,7 +491,7 @@ abstract class DrupalLocalStreamWrapper implements DrupalStreamWrapperInterface
    * @return
    *   TRUE on success.
    *
-   * @see http://php.net/manual/en/streamwrapper.stream-seek.php
+   * @see http://php.net/manual/streamwrapper.stream-seek.php
    */
   public function stream_seek($offset, $whence) {
     // fseek returns 0 on success and -1 on a failure.
@@ -506,7 +505,7 @@ abstract class DrupalLocalStreamWrapper implements DrupalStreamWrapperInterface
    * @return
    *   TRUE if data was successfully stored (or there was no data to store).
    *
-   * @see http://php.net/manual/en/streamwrapper.stream-flush.php
+   * @see http://php.net/manual/streamwrapper.stream-flush.php
    */
   public function stream_flush() {
     return fflush($this->handle);
@@ -518,7 +517,7 @@ abstract class DrupalLocalStreamWrapper implements DrupalStreamWrapperInterface
    * @return
    *   The current offset in bytes from the beginning of file.
    *
-   * @see http://php.net/manual/en/streamwrapper.stream-tell.php
+   * @see http://php.net/manual/streamwrapper.stream-tell.php
    */
   public function stream_tell() {
     return ftell($this->handle);
@@ -531,7 +530,7 @@ abstract class DrupalLocalStreamWrapper implements DrupalStreamWrapperInterface
    *   An array with file status, or FALSE in case of an error - see fstat()
    *   for a description of this array.
    *
-   * @see http://php.net/manual/en/streamwrapper.stream-stat.php
+   * @see http://php.net/manual/streamwrapper.stream-stat.php
    */
   public function stream_stat() {
     return fstat($this->handle);
@@ -543,12 +542,161 @@ abstract class DrupalLocalStreamWrapper implements DrupalStreamWrapperInterface
    * @return
    *   TRUE if stream was successfully closed.
    *
-   * @see http://php.net/manual/en/streamwrapper.stream-close.php
+   * @see http://php.net/manual/streamwrapper.stream-close.php
    */
   public function stream_close() {
     return fclose($this->handle);
   }
 
+  /**
+   * Sets metadata on the stream.
+   *
+   * WARNING: Do not call this method directly! It will be called internally by
+   * PHP itself when one of the following functions is called on a stream URL:
+   *
+   * @param string $uri
+   *   A string containing the URI to the file to set metadata on.
+   * @param int $option
+   *   One of:
+   *   - STREAM_META_TOUCH: The method was called in response to touch().
+   *   - STREAM_META_OWNER_NAME: The method was called in response to chown()
+   *     with string parameter.
+   *   - STREAM_META_OWNER: The method was called in response to chown().
+   *   - STREAM_META_GROUP_NAME: The method was called in response to chgrp().
+   *   - STREAM_META_GROUP: The method was called in response to chgrp().
+   *   - STREAM_META_ACCESS: The method was called in response to chmod().
+   * @param mixed $value
+   *   If option is:
+   *   - STREAM_META_TOUCH: Array consisting of two arguments of the touch()
+   *     function.
+   *   - STREAM_META_OWNER_NAME or STREAM_META_GROUP_NAME: The name of the owner
+   *     user/group as string.
+   *   - STREAM_META_OWNER or STREAM_META_GROUP: The value of the owner
+   *     user/group as integer.
+   *   - STREAM_META_ACCESS: The argument of the chmod() as integer.
+   *
+   * @return bool
+   *   Returns TRUE on success or FALSE on failure. If $option is not
+   *   implemented, FALSE should be returned.
+   *
+   * @see touch()
+   * @see chmod()
+   * @see chown()
+   * @see chgrp()
+   * @link http://php.net/manual/streamwrapper.stream-metadata.php
+   */
+  public function stream_metadata($uri, $option, $value) {
+    $target = $this->getLocalPath($uri);
+    $return = FALSE;
+    switch ($option) {
+      case STREAM_META_TOUCH:
+        if (!empty($value)) {
+          $return = touch($target, $value[0], $value[1]);
+        }
+        else {
+          $return = touch($target);
+        }
+        break;
+
+      case STREAM_META_OWNER_NAME:
+      case STREAM_META_OWNER:
+        $return = chown($target, $value);
+        break;
+
+      case STREAM_META_GROUP_NAME:
+      case STREAM_META_GROUP:
+        $return = chgrp($target, $value);
+        break;
+
+      case STREAM_META_ACCESS:
+        $return = chmod($target, $value);
+        break;
+    }
+    if ($return) {
+      // For convenience clear the file status cache of the underlying file,
+      // since metadata operations are often followed by file status checks.
+      clearstatcache(TRUE, $target);
+    }
+    return $return;
+  }
+
+  /**
+   * Truncate stream.
+   *
+   * Will respond to truncation; e.g., through ftruncate().
+   *
+   * @param int $new_size
+   *   The new size.
+   *
+   * @return bool
+   *   TRUE on success, FALSE otherwise.
+   */
+  public function stream_truncate($new_size) {
+    return ftruncate($this->handle, $new_size);
+  }
+
+  /**
+   * Retrieve the underlying stream resource.
+   *
+   * This method is called in response to stream_select().
+   *
+   * @param int $cast_as
+   *   Can be STREAM_CAST_FOR_SELECT when stream_select() is calling
+   *   stream_cast() or STREAM_CAST_AS_STREAM when stream_cast() is called for
+   *   other uses.
+   *
+   * @return resource|false
+   *   The underlying stream resource or FALSE if stream_select() is not
+   *   supported.
+   *
+   * @see stream_select()
+   * @link http://php.net/manual/streamwrapper.stream-cast.php
+   */
+  public function stream_cast($cast_as) {
+    return $this->handle ? $this->handle : FALSE;
+  }
+
+  /**
+   * Change stream options.
+   *
+   * This method is called to set options on the stream.
+   *
+   * Since Windows systems do not allow it and it is not needed for most use
+   * cases anyway, this method is not supported on local files and will trigger
+   * an error and return false. If needed, custom subclasses can provide
+   * OS-specific implementations for advanced use cases.
+   *
+   * @param int $option
+   *   One of:
+   *   - STREAM_OPTION_BLOCKING: The method was called in response to
+   *     stream_set_blocking().
+   *   - STREAM_OPTION_READ_TIMEOUT: The method was called in response to
+   *     stream_set_timeout().
+   *   - STREAM_OPTION_WRITE_BUFFER: The method was called in response to
+   *     stream_set_write_buffer().
+   * @param int $arg1
+   *   If option is:
+   *   - STREAM_OPTION_BLOCKING: The requested blocking mode:
+   *     - 1 means blocking.
+   *     - 0 means not blocking.
+   *   - STREAM_OPTION_READ_TIMEOUT: The timeout in seconds.
+   *   - STREAM_OPTION_WRITE_BUFFER: The buffer mode, STREAM_BUFFER_NONE or
+   *     STREAM_BUFFER_FULL.
+   * @param int $arg2
+   *   If option is:
+   *   - STREAM_OPTION_BLOCKING: This option is not set.
+   *   - STREAM_OPTION_READ_TIMEOUT: The timeout in microseconds.
+   *   - STREAM_OPTION_WRITE_BUFFER: The requested buffer size.
+   *
+   * @return bool
+   *   TRUE on success, FALSE otherwise. If $option is not implemented, FALSE
+   *   should be returned.
+   */
+  public function stream_set_option($option, $arg1, $arg2) {
+    trigger_error('stream_set_option() not supported for local file based stream wrappers', E_USER_WARNING);
+    return FALSE;
+  }
+
   /**
    * Support for unlink().
    *
@@ -558,7 +706,7 @@ abstract class DrupalLocalStreamWrapper implements DrupalStreamWrapperInterface
    * @return
    *   TRUE if resource was successfully deleted.
    *
-   * @see http://php.net/manual/en/streamwrapper.unlink.php
+   * @see http://php.net/manual/streamwrapper.unlink.php
    */
   public function unlink($uri) {
     $this->uri = $uri;
@@ -576,7 +724,7 @@ abstract class DrupalLocalStreamWrapper implements DrupalStreamWrapperInterface
    * @return
    *   TRUE if file was successfully renamed.
    *
-   * @see http://php.net/manual/en/streamwrapper.rename.php
+   * @see http://php.net/manual/streamwrapper.rename.php
    */
   public function rename($from_uri, $to_uri) {
     return rename($this->getLocalPath($from_uri), $this->getLocalPath($to_uri));
@@ -622,7 +770,7 @@ abstract class DrupalLocalStreamWrapper implements DrupalStreamWrapperInterface
    * @return
    *   TRUE if directory was successfully created.
    *
-   * @see http://php.net/manual/en/streamwrapper.mkdir.php
+   * @see http://php.net/manual/streamwrapper.mkdir.php
    */
   public function mkdir($uri, $mode, $options) {
     $this->uri = $uri;
@@ -654,7 +802,7 @@ abstract class DrupalLocalStreamWrapper implements DrupalStreamWrapperInterface
    * @return
    *   TRUE if directory was successfully removed.
    *
-   * @see http://php.net/manual/en/streamwrapper.rmdir.php
+   * @see http://php.net/manual/streamwrapper.rmdir.php
    */
   public function rmdir($uri, $options) {
     $this->uri = $uri;
@@ -678,7 +826,7 @@ abstract class DrupalLocalStreamWrapper implements DrupalStreamWrapperInterface
    *   An array with file status, or FALSE in case of an error - see fstat()
    *   for a description of this array.
    *
-   * @see http://php.net/manual/en/streamwrapper.url-stat.php
+   * @see http://php.net/manual/streamwrapper.url-stat.php
    */
   public function url_stat($uri, $flags) {
     $this->uri = $uri;
@@ -704,7 +852,7 @@ abstract class DrupalLocalStreamWrapper implements DrupalStreamWrapperInterface
    * @return
    *   TRUE on success.
    *
-   * @see http://php.net/manual/en/streamwrapper.dir-opendir.php
+   * @see http://php.net/manual/streamwrapper.dir-opendir.php
    */
   public function dir_opendir($uri, $options) {
     $this->uri = $uri;
@@ -719,7 +867,7 @@ abstract class DrupalLocalStreamWrapper implements DrupalStreamWrapperInterface
    * @return
    *   The next filename, or FALSE if there are no more files in the directory.
    *
-   * @see http://php.net/manual/en/streamwrapper.dir-readdir.php
+   * @see http://php.net/manual/streamwrapper.dir-readdir.php
    */
   public function dir_readdir() {
     return readdir($this->handle);
@@ -731,7 +879,7 @@ abstract class DrupalLocalStreamWrapper implements DrupalStreamWrapperInterface
    * @return
    *   TRUE on success.
    *
-   * @see http://php.net/manual/en/streamwrapper.dir-rewinddir.php
+   * @see http://php.net/manual/streamwrapper.dir-rewinddir.php
    */
   public function dir_rewinddir() {
     rewinddir($this->handle);
@@ -747,7 +895,7 @@ abstract class DrupalLocalStreamWrapper implements DrupalStreamWrapperInterface
    * @return
    *   TRUE on success.
    *
-   * @see http://php.net/manual/en/streamwrapper.dir-closedir.php
+   * @see http://php.net/manual/streamwrapper.dir-closedir.php
    */
   public function dir_closedir() {
     closedir($this->handle);
@@ -788,8 +936,6 @@ class DrupalPublicStreamWrapper extends DrupalLocalStreamWrapper {
  *
  * Provides support for storing privately accessible files with the Drupal file
  * interface.
- *
- * Extends DrupalPublicStreamWrapper.
  */
 class DrupalPrivateStreamWrapper extends DrupalLocalStreamWrapper {
   /**

includes/tablesort.inc

@@ -46,10 +46,9 @@ class TableSort extends SelectQueryExtender {
       // Based on code from db_escape_table(), but this can also contain a dot.
       $field = preg_replace('/[^A-Za-z0-9_.]+/', '', $ts['sql']);
 
-      // Sort order can only be ASC or DESC.
-      $sort = drupal_strtoupper($ts['sort']);
-      $sort = in_array($sort, array('ASC', 'DESC')) ? $sort : '';
-      $this->orderBy($field, $sort);
+      // orderBy() will ensure that only ASC/DESC values are accepted, so we
+      // don't need to sanitize that here.
+      $this->orderBy($field, $ts['sort']);
     }
     return $this;
   }

includes/theme.inc

@@ -508,7 +508,7 @@ class ThemeRegistry Extends DrupalCacheArray {
  *   themes/bartik.
  *
  * @see theme()
- * @see _theme_process_registry()
+ * @see _theme_build_registry()
  * @see hook_theme()
  * @see list_themes()
  */
@@ -869,11 +869,18 @@ function drupal_find_base_themes($themes, $key, $used_keys = array()) {
 /**
  * Generates themed output.
  *
- * All requests for themed output must go through this function. It examines
- * the request and routes it to the appropriate
+ * All requests for themed output must go through this function (however,
+ * calling the theme() function directly is strongly discouraged - see next
+ * paragraph). It examines the request and routes it to the appropriate
  * @link themeable theme function or template @endlink, by checking the theme
  * registry.
  *
+ * Avoid calling this function directly. It is preferable to replace direct
+ * calls to the theme() function with calls to drupal_render() by passing a
+ * render array with a #theme key to drupal_render(), which in turn calls
+ * theme().
+ *
+ * @section sec_theme_hooks Theme Hooks
  * Most commonly, the first argument to this function is the name of the theme
  * hook. For instance, to theme a taxonomy term, the theme hook name is
  * 'taxonomy_term'. Modules register theme hooks within a hook_theme()
@@ -885,6 +892,7 @@ function drupal_find_base_themes($themes, $key, $used_keys = array()) {
  * underscores changed to hyphens, so for the 'taxonomy_term' theme hook, the
  * default template is 'taxonomy-term.tpl.php'.
  *
+ * @subsection sub_overriding_theme_hooks Overriding Theme Hooks
  * Themes may also register new theme hooks within a hook_theme()
  * implementation, but it is more common for themes to override default
  * implementations provided by modules than to register entirely new theme
@@ -897,6 +905,7 @@ function drupal_find_base_themes($themes, $key, $used_keys = array()) {
  * rendering engine, it overrides the default implementation of the 'page' theme
  * hook by containing a 'page.tpl.php' file within its folder structure).
  *
+ * @subsection sub_preprocess_templates Preprocessing for Template Files
  * If the implementation is a template file, several functions are called
  * before the template file is invoked, to modify the $variables array. These
  * fall into the "preprocessing" phase and the "processing" phase, and are
@@ -945,12 +954,14 @@ function drupal_find_base_themes($themes, $key, $used_keys = array()) {
  * - THEME_process_HOOK(&$variables):  Allows the theme to process the
  *   variables specific to the theme hook.
  *
+ * @subsection sub_preprocess_theme_funcs Preprocessing for Theme Functions
  * If the implementation is a function, only the theme-hook-specific preprocess
  * and process functions (the ones ending in _HOOK) are called from the
  * list above. This is because theme hooks with function implementations
  * need to be fast, and calling the non-theme-hook-specific preprocess and
  * process functions for them would incur a noticeable performance penalty.
  *
+ * @subsection sub_alternate_suggestions Suggesting Alternate Hooks
  * There are two special variables that these preprocess and process functions
  * can set: 'theme_hook_suggestion' and 'theme_hook_suggestions'. These will be
  * merged together to form a list of 'suggested' alternate theme hooks to use,
@@ -992,6 +1003,7 @@ function drupal_find_base_themes($themes, $key, $used_keys = array()) {
  * @return
  *   An HTML string representing the themed output.
  *
+ * @see drupal_render()
  * @see themeable
  * @see hook_theme()
  * @see template_preprocess()
@@ -1017,6 +1029,7 @@ function theme($hook, $variables = array()) {
     }
     $hook = $candidate;
   }
+  $theme_hook_original = $hook;
 
   // If there's no implementation, check for more generic fallbacks. If there's
   // still no implementation, log an error and return an empty string.
@@ -1078,6 +1091,8 @@ function theme($hook, $variables = array()) {
     $variables += array($info['render element'] => array());
   }
 
+  $variables['theme_hook_original'] = $theme_hook_original;
+
   // Invoke the variable processors, if any. The processors may specify
   // alternate suggestions for which hook's template/function to use. If the
   // hook is a suggestion of a base hook, invoke the variable processors of
@@ -1186,7 +1201,12 @@ function theme($hook, $variables = array()) {
     if (isset($info['path'])) {
       $template_file = $info['path'] . '/' . $template_file;
     }
-    $output = $render_function($template_file, $variables);
+    if (variable_get('theme_debug', FALSE)) {
+      $output = _theme_render_template_debug($render_function, $template_file, $variables, $extension);
+    }
+    else {
+      $output = $render_function($template_file, $variables);
+    }
   }
 
   // restore path_to_theme()
@@ -1228,6 +1248,7 @@ function path_to_theme() {
 function drupal_find_theme_functions($cache, $prefixes) {
   $implementations = array();
   $functions = get_defined_functions();
+  $theme_functions = preg_grep('/^(' . implode(')|(', $prefixes) . ')_/', $functions['user']);
 
   foreach ($cache as $hook => $info) {
     foreach ($prefixes as $prefix) {
@@ -1244,7 +1265,7 @@ function drupal_find_theme_functions($cache, $prefixes) {
       // intermediary suggestion.
       $pattern = isset($info['pattern']) ? $info['pattern'] : ($hook . '__');
       if (!isset($info['base hook']) && !empty($pattern)) {
-        $matches = preg_grep('/^' . $prefix . '_' . $pattern . '/', $functions['user']);
+        $matches = preg_grep('/^' . $prefix . '_' . $pattern . '/', $theme_functions);
         if ($matches) {
           foreach ($matches as $match) {
             $new_hook = substr($match, strlen($prefix) + 1);
@@ -1508,6 +1529,72 @@ function theme_render_template($template_file, $variables) {
   return ob_get_clean();
 }
 
+/**
+ * Renders a template for any engine.
+ *
+ * Includes the possibility to get debug output by setting the
+ * theme_debug variable to TRUE.
+ *
+ * @param string $template_function
+ *   The function to call for rendering the template.
+ * @param string $template_file
+ *   The filename of the template to render.
+ * @param array $variables
+ *   A keyed array of variables that will appear in the output.
+ * @param string $extension
+ *   The extension used by the theme engine for template files.
+ *
+ * @return string
+ *   The output generated by the template including debug information.
+ */
+function _theme_render_template_debug($template_function, $template_file, $variables, $extension) {
+  $output = array(
+    'debug_prefix' => '',
+    'debug_info' => '',
+    'rendered_markup' => call_user_func($template_function, $template_file, $variables),
+    'debug_suffix' => '',
+  );
+  $output['debug_prefix'] .= "\n\n<!-- THEME DEBUG -->";
+  $output['debug_prefix'] .= "\n<!-- CALL: theme('" . check_plain($variables['theme_hook_original']) . "') -->";
+  // If there are theme suggestions, reverse the array so more specific
+  // suggestions are shown first.
+  if (!empty($variables['theme_hook_suggestions'])) {
+    $variables['theme_hook_suggestions'] = array_reverse($variables['theme_hook_suggestions']);
+  }
+  // Add debug output for directly called suggestions like
+  // '#theme' => 'comment__node__article'.
+  if (strpos($variables['theme_hook_original'], '__') !== FALSE) {
+    $derived_suggestions[] = $hook = $variables['theme_hook_original'];
+    while ($pos = strrpos($hook, '__')) {
+      $hook = substr($hook, 0, $pos);
+      $derived_suggestions[] = $hook;
+    }
+    // Get the value of the base hook (last derived suggestion) and append it
+    // to the end of all theme suggestions.
+    $base_hook = array_pop($derived_suggestions);
+    $variables['theme_hook_suggestions'] = array_merge($derived_suggestions, $variables['theme_hook_suggestions']);
+    $variables['theme_hook_suggestions'][] = $base_hook;
+  }
+  if (!empty($variables['theme_hook_suggestions'])) {
+    $current_template = basename($template_file);
+    $suggestions = $variables['theme_hook_suggestions'];
+    // Only add the original theme hook if it wasn't a directly called
+    // suggestion.
+    if (strpos($variables['theme_hook_original'], '__') === FALSE) {
+      $suggestions[] = $variables['theme_hook_original'];
+    }
+    foreach ($suggestions as &$suggestion) {
+      $template = strtr($suggestion, '_', '-') . $extension;
+      $prefix = ($template == $current_template) ? 'x' : '*';
+      $suggestion = $prefix . ' ' . $template;
+    }
+    $output['debug_info'] .= "\n<!-- FILE NAME SUGGESTIONS:\n   " . check_plain(implode("\n   ", $suggestions)) . "\n-->";
+  }
+  $output['debug_info'] .= "\n<!-- BEGIN OUTPUT from '" . check_plain($template_file) . "' -->\n";
+  $output['debug_suffix'] .= "\n<!-- END OUTPUT from '" . check_plain($template_file) . "' -->\n\n";
+  return implode('', $output);
+}
+
 /**
  * Enables a given list of themes.
  *
@@ -1605,7 +1692,7 @@ function theme_status_messages($variables) {
       $output .= " </ul>\n";
     }
     else {
-      $output .= $messages[0];
+      $output .= reset($messages);
     }
     $output .= "</div>\n";
   }
@@ -1624,11 +1711,29 @@ function theme_status_messages($variables) {
  * copy if none of the enabled modules or the active theme implement any
  * preprocess or process functions or override this theme implementation.
  *
- * @param $variables
- *   An associative array containing the keys 'text', 'path', and 'options'.
- *   See the l() function for information about these variables.
+ * @param array $variables
+ *   An associative array containing the keys:
+ *   - text: The text of the link.
+ *   - path: The internal path or external URL being linked to. It is used as
+ *     the $path parameter of the url() function.
+ *   - options: (optional) An array that defaults to empty, but can contain:
+ *     - attributes: Can contain optional attributes:
+ *       - class: must be declared in an array. Example: 'class' =>
+ *         array('class_name1','class_name2').
+ *       - title: must be a string. Example: 'title' => 'Example title'
+ *       - Others are more flexible as long as they work with
+ *         drupal_attributes($variables['options']['attributes]).
+ *     - html: Boolean flag that tells whether text contains HTML or plain
+ *       text. If set to TRUE, the text value will not be sanitized so the
+         calling function must ensure that it already contains safe HTML.
+ *   The elements $variables['options']['attributes'] and
+ *   $variables['options']['html'] are used in this function similarly to the
+ *   way that $options['attributes'] and $options['html'] are used in l().
+ *   The link itself is built by the url() function, which takes
+ *   $variables['path'] and $variables['options'] as arguments.
  *
  * @see l()
+ * @see url()
  */
 function theme_link($variables) {
   return '<a href="' . check_plain(url($variables['path'], $variables['options'])) . '"' . drupal_attributes($variables['options']['attributes']) . '>' . ($variables['options']['html'] ? $variables['text'] : check_plain($variables['text'])) . '</a>';
@@ -1671,15 +1776,13 @@ function theme_link($variables) {
  *     http://www.w3.org/TR/WCAG-TECHS/H42.html for more information.
  */
 function theme_links($variables) {
-  $links = $variables['links'];
-  $attributes = $variables['attributes'];
+  $links = (array) $variables['links'];
+  $attributes = (array) $variables['attributes'];
   $heading = $variables['heading'];
   global $language_url;
   $output = '';
 
-  if (count($links) > 0) {
-    $output = '';
-
+  if (!empty($links)) {
     // Treat the heading first if it is present to prepend it to the
     // list of links.
     if (!empty($heading)) {
@@ -1707,7 +1810,8 @@ function theme_links($variables) {
     foreach ($links as $key => $link) {
       $class = array($key);
 
-      // Add first, last and active classes to the list of links to help out themers.
+      // Add first, last and active classes to the list of links to help out
+      // themers.
       if ($i == 1) {
         $class[] = 'first';
       }
@@ -1725,7 +1829,8 @@ function theme_links($variables) {
         $output .= l($link['title'], $link['href'], $link);
       }
       elseif (!empty($link['title'])) {
-        // Some links are actually not links, but we wrap these in <span> for adding title and class attributes.
+        // Some links are actually not links, but we wrap these in <span> for
+        // adding title and class attributes.
         if (empty($link['html'])) {
           $link['title'] = check_plain($link['title']);
         }
@@ -1806,7 +1911,7 @@ function theme_breadcrumb($variables) {
 /**
  * Returns HTML for a table.
  *
- * @param $variables
+ * @param array $variables
  *   An associative array containing:
  *   - header: An array containing the table headers. Each element of the array
  *     can be either a localized string or an associative array with the
@@ -1814,7 +1919,9 @@ function theme_breadcrumb($variables) {
  *     - "data": The localized title of the table column.
  *     - "field": The database field represented in the table column (required
  *       if user is to be able to sort on this column).
- *     - "sort": A default sort order for this column ("asc" or "desc").
+ *     - "sort": A default sort order for this column ("asc" or "desc"). Only
+ *       one column should be given a default sort order because table sorting
+ *       only applies to one column at a time.
  *     - Any HTML attributes, such as "colspan", to apply to the column header
  *       cell.
  *   - rows: An array of table rows. Every row is an array of cells, or an
@@ -1841,6 +1948,11 @@ function theme_breadcrumb($variables) {
  *       )
  *     );
  *     @endcode
+ *   - footer: An array of table rows which will be printed within a <tfoot>
+ *     tag, in the same format as the rows element (see above).
+ *     The structure is the same the one defined for the "rows" key except
+ *     that the no_striping boolean has no effect, there is no rows striping
+ *     for the table footer.
  *   - attributes: An array of HTML attributes to apply to the table tag.
  *   - caption: A localized string to use for the <caption> tag.
  *   - colgroups: An array of column groups. Each element of the array can be
@@ -1877,8 +1989,11 @@ function theme_breadcrumb($variables) {
  *   - sticky: Use a "sticky" table header.
  *   - empty: The message to display in an extra row if table does not have any
  *     rows.
+ *
+ * @return string
+ *   The HTML output.
  */
-function theme_table($variables) {
+function theme_table(array $variables) {
   $header = $variables['header'];
   $rows = $variables['rows'];
   $attributes = $variables['attributes'];
@@ -1888,7 +2003,7 @@ function theme_table($variables) {
   $empty = $variables['empty'];
 
   // Add sticky headers, if applicable.
-  if (count($header) && $sticky) {
+  if (!empty($header) && $sticky) {
     drupal_add_js('misc/tableheader.js');
     // Add 'sticky-enabled' class to the table to identify it for JS.
     // This is needed to target tables constructed by this function.
@@ -1902,7 +2017,7 @@ function theme_table($variables) {
   }
 
   // Format the table columns:
-  if (count($colgroups)) {
+  if (!empty($colgroups)) {
     foreach ($colgroups as $number => $colgroup) {
       $attributes = array();
 
@@ -1937,66 +2052,94 @@ function theme_table($variables) {
   }
 
   // Add the 'empty' row message if available.
-  if (!count($rows) && $empty) {
+  if (empty($rows) && $empty) {
     $header_count = 0;
-    foreach ($header as $header_cell) {
-      if (is_array($header_cell)) {
-        $header_count += isset($header_cell['colspan']) ? $header_cell['colspan'] : 1;
-      }
-      else {
-        $header_count++;
+    if (!empty($header)) {
+      foreach ($header as $header_cell) {
+        if (is_array($header_cell)) {
+          $header_count += isset($header_cell['colspan']) ?
+            $header_cell['colspan'] : 1;
+        }
+        else {
+          $header_count++;
+        }
       }
     }
-    $rows[] = array(array('data' => $empty, 'colspan' => $header_count, 'class' => array('empty', 'message')));
+    $rows[] = array(
+      array(
+        'data' => $empty,
+        'colspan' => $header_count,
+        'class' => array(
+          'empty',
+          'message'
+        ),
+      ),
+    );
   }
 
-  // Format the table header:
-  if (count($header)) {
+  // Format the table header.
+  if (!empty($header)) {
     $ts = tablesort_init($header);
     // HTML requires that the thead tag has tr tags in it followed by tbody
     // tags. Using ternary operator to check and see if we have any rows.
-    $output .= (count($rows) ? ' <thead><tr>' : ' <tr>');
+    $output .= (!empty($rows) ? ' <thead><tr>' : ' <tr>');
     foreach ($header as $cell) {
       $cell = tablesort_header($cell, $header, $ts);
       $output .= _theme_table_cell($cell, TRUE);
     }
-    // Using ternary operator to close the tags based on whether or not there are rows
-    $output .= (count($rows) ? " </tr></thead>\n" : "</tr>\n");
+    // Using ternary operator to close the tags based on whether
+    // or not there are rows.
+    $output .= (!empty($rows) ? " </tr></thead>\n" : "</tr>\n");
   }
   else {
     $ts = array();
   }
 
-  // Format the table rows:
-  if (count($rows)) {
-    $output .= "<tbody>\n";
+  // Format the table and footer rows.
+  $sections = array();
+
+  if (!empty($rows)) {
+    $sections['tbody'] = $rows;
+  }
+
+  if (!empty($variables['footer'])) {
+    $sections['tfoot'] = $variables['footer'];
+  }
+
+  // tbody and tfoot have the same structure and are built using the same
+  // procedure.
+  foreach ($sections as $tag => $content) {
+    $output .= "<" . $tag . ">\n";
     $flip = array('even' => 'odd', 'odd' => 'even');
     $class = 'even';
-    foreach ($rows as $number => $row) {
-      $attributes = array();
+    $default_no_striping = ($tag === 'tfoot');
 
-      // Check if we're dealing with a simple or complex row
+    foreach ($content as $number => $row) {
+      // Check if we're dealing with a simple or complex row.
       if (isset($row['data'])) {
-        foreach ($row as $key => $value) {
-          if ($key == 'data') {
-            $cells = $value;
-          }
-          else {
-            $attributes[$key] = $value;
-          }
-        }
+        $cells = $row['data'];
+        $no_striping = isset($row['no_striping']) ?
+          $row['no_striping'] : $default_no_striping;
+
+        // Set the attributes array and exclude 'data' and 'no_striping'.
+        $attributes = $row;
+        unset($attributes['data']);
+        unset($attributes['no_striping']);
       }
       else {
         $cells = $row;
+        $attributes = array();
+        $no_striping = $default_no_striping;
       }
-      if (count($cells)) {
-        // Add odd/even class
-        if (empty($row['no_striping'])) {
+
+      if (!empty($cells)) {
+        // Add odd/even class.
+        if (!$no_striping) {
           $class = $flip[$class];
           $attributes['class'][] = $class;
         }
 
-        // Build row
+        // Build row.
         $output .= ' <tr' . drupal_attributes($attributes) . '>';
         $i = 0;
         foreach ($cells as $cell) {
@@ -2006,10 +2149,12 @@ function theme_table($variables) {
         $output .= " </tr>\n";
       }
     }
-    $output .= "</tbody>\n";
+
+    $output .= "</" . $tag . ">\n";
   }
 
   $output .= "</table>\n";
+
   return $output;
 }
 
@@ -2533,10 +2678,13 @@ function template_preprocess_page(&$variables) {
   // Move some variables to the top level for themer convenience and template cleanliness.
   $variables['show_messages'] = $variables['page']['#show_messages'];
 
-  foreach (system_region_list($GLOBALS['theme']) as $region_key => $region_name) {
+  foreach (system_region_list($GLOBALS['theme'], REGIONS_ALL, FALSE) as $region_key) {
     if (!isset($variables['page'][$region_key])) {
       $variables['page'][$region_key] = array();
     }
+    if ($region_content = drupal_get_region_content($region_key)) {
+      $variables['page'][$region_key][]['#markup'] = $region_content;
+    }
   }
 
   // Set up layout variable.

includes/unicode.inc

@@ -116,11 +116,15 @@ function _unicode_check() {
   if (ini_get('mbstring.encoding_translation') != 0) {
     return array(UNICODE_ERROR, $t('Multibyte string input conversion in PHP is active and must be disabled. Check the php.ini <em>mbstring.encoding_translation</em> setting. Please refer to the <a href="@url">PHP mbstring documentation</a> for more information.', array('@url' => 'http://www.php.net/mbstring')));
   }
-  if (ini_get('mbstring.http_input') != 'pass') {
-    return array(UNICODE_ERROR, $t('Multibyte string input conversion in PHP is active and must be disabled. Check the php.ini <em>mbstring.http_input</em> setting. Please refer to the <a href="@url">PHP mbstring documentation</a> for more information.', array('@url' => 'http://www.php.net/mbstring')));
-  }
-  if (ini_get('mbstring.http_output') != 'pass') {
-    return array(UNICODE_ERROR, $t('Multibyte string output conversion in PHP is active and must be disabled. Check the php.ini <em>mbstring.http_output</em> setting. Please refer to the <a href="@url">PHP mbstring documentation</a> for more information.', array('@url' => 'http://www.php.net/mbstring')));
+  // mbstring.http_input and mbstring.http_output are deprecated and empty by
+  // default in PHP 5.6.
+  if (version_compare(PHP_VERSION, '5.6.0') == -1) {
+    if (ini_get('mbstring.http_input') != 'pass') {
+      return array(UNICODE_ERROR, $t('Multibyte string input conversion in PHP is active and must be disabled. Check the php.ini <em>mbstring.http_input</em> setting. Please refer to the <a href="@url">PHP mbstring documentation</a> for more information.', array('@url' => 'http://www.php.net/mbstring')));
+    }
+    if (ini_get('mbstring.http_output') != 'pass') {
+      return array(UNICODE_ERROR, $t('Multibyte string output conversion in PHP is active and must be disabled. Check the php.ini <em>mbstring.http_output</em> setting. Please refer to the <a href="@url">PHP mbstring documentation</a> for more information.', array('@url' => 'http://www.php.net/mbstring')));
+    }
   }
 
   // Set appropriate configuration

includes/update.inc

@@ -795,6 +795,14 @@ function update_fix_d7_requirements() {
 function update_fix_d7_install_profile() {
   $profile = drupal_get_profile();
 
+  // 'Default' profile has been renamed to 'Standard' in D7.
+  // We change the profile here to prevent a broken record in the system table.
+  // See system_update_7049().
+  if ($profile == 'default') {
+    $profile = 'standard';
+    variable_set('install_profile', $profile);
+  }
+
   $results = db_select('system', 's')
     ->fields('s', array('name', 'schema_version'))
     ->condition('name', $profile)
@@ -908,6 +916,8 @@ function update_get_d6_session_name() {
 }
 
 /**
+ * Implements callback_batch_operation().
+ *
  * Performs one update and stores the results for display on the results page.
  *
  * If an update function completes successfully, it should return a message
@@ -1078,6 +1088,8 @@ function update_batch($start, $redirect = NULL, $url = NULL, $batch = array(), $
 }
 
 /**
+ * Implements callback_batch_finished().
+ *
  * Finishes the update process and stores the results for eventual display.
  *
  * After the updates run, all caches are flushed. The update results are

includes/xmlrpc.inc

@@ -178,7 +178,41 @@ function xmlrpc_message_parse($xmlrpc_message) {
   xml_set_element_handler($xmlrpc_message->_parser, 'xmlrpc_message_tag_open', 'xmlrpc_message_tag_close');
   xml_set_character_data_handler($xmlrpc_message->_parser, 'xmlrpc_message_cdata');
   xmlrpc_message_set($xmlrpc_message);
-  if (!xml_parse($xmlrpc_message->_parser, $xmlrpc_message->message)) {
+
+  // Strip XML declaration.
+  $header = preg_replace('/<\?xml.*?\?'.'>/s', '', substr($xmlrpc_message->message, 0, 100), 1);
+  $xml = trim(substr_replace($xmlrpc_message->message, $header, 0, 100));
+  if ($xml == '') {
+    return FALSE;
+  }
+  // Strip DTD.
+  $header = preg_replace('/^<!DOCTYPE[^>]*+>/i', '', substr($xml, 0, 200), 1);
+  $xml = trim(substr_replace($xml, $header, 0, 200));
+  if ($xml == '') {
+    return FALSE;
+  }
+  // Confirm the XML now starts with a valid root tag. A root tag can end in [> \t\r\n]
+  $root_tag = substr($xml, 0, strcspn(substr($xml, 0, 20), "> \t\r\n"));
+  // Reject a second DTD.
+  if (strtoupper($root_tag) == '<!DOCTYPE') {
+    return FALSE;
+  }
+  if (!in_array($root_tag, array('<methodCall', '<methodResponse', '<fault'))) {
+    return FALSE;
+  }
+  // Skip parsing if there is an unreasonably large number of tags.
+  try {
+    $dom = new DOMDocument();
+    @$dom->loadXML($xml);
+    if ($dom->getElementsByTagName('*')->length > variable_get('xmlrpc_message_maximum_tag_count', 30000)) {
+      return FALSE;
+    }
+  }
+  catch (Exception $e) {
+    return FALSE;
+  }
+
+  if (!xml_parse($xmlrpc_message->_parser, $xml)) {
     return FALSE;
   }
   xml_parser_free($xmlrpc_message->_parser);

includes/xmlrpcs.inc

@@ -9,7 +9,9 @@
  * Invokes XML-RPC methods on this server.
  *
  * @param array $callbacks
- *   Array of external XML-RPC method names with the callbacks they map to.
+ *   Either an associative array of external XML-RPC method names as keys with
+ *   the callbacks they map to as values, or a more complex structure
+ *   describing XML-RPC callbacks as returned from hook_xmlrpc().
  */
 function xmlrpc_server($callbacks) {
   $xmlrpc_server = new stdClass();
@@ -262,6 +264,10 @@ function xmlrpc_server_call($xmlrpc_server, $methodname, $args) {
  */
 function xmlrpc_server_multicall($methodcalls) {
   // See http://www.xmlrpc.com/discuss/msgReader$1208
+  // To avoid multicall expansion attacks, limit the number of duplicate method
+  // calls allowed with a default of 1. Set to -1 for unlimited.
+  $duplicate_method_limit = variable_get('xmlrpc_multicall_duplicate_method_limit', 1);
+  $method_count = array();
   $return = array();
   $xmlrpc_server = xmlrpc_server_get();
   foreach ($methodcalls as $call) {
@@ -271,10 +277,14 @@ function xmlrpc_server_multicall($methodcalls) {
       $ok = FALSE;
     }
     $method = $call['methodName'];
+    $method_count[$method] = isset($method_count[$method]) ? $method_count[$method] + 1 : 1;
     $params = $call['params'];
     if ($method == 'system.multicall') {
       $result = xmlrpc_error(-32600, t('Recursive calls to system.multicall are forbidden.'));
     }
+    elseif ($duplicate_method_limit > 0 && $method_count[$method] > $duplicate_method_limit) {
+      $result = xmlrpc_error(-156579, t('Too many duplicate method calls in system.multicall.'));
+    }
     elseif ($ok) {
       $result = xmlrpc_server_call($xmlrpc_server, $method, $params);
     }

misc/ajax.js

@@ -14,6 +14,8 @@
 
 Drupal.ajax = Drupal.ajax || {};
 
+Drupal.settings.urlIsAjaxTrusted = Drupal.settings.urlIsAjaxTrusted || {};
+
 /**
  * Attaches the Ajax behavior to each Ajax form element.
  */
@@ -130,6 +132,11 @@ Drupal.ajax = function (base, element, element_settings) {
   // 5. /nojs# - Followed by a fragment.
   //      E.g.: path/nojs#myfragment
   this.url = element_settings.url.replace(/\/nojs(\/|$|\?|&|#)/g, '/ajax$1');
+  // If the 'nojs' version of the URL is trusted, also trust the 'ajax' version.
+  if (Drupal.settings.urlIsAjaxTrusted[element_settings.url]) {
+    Drupal.settings.urlIsAjaxTrusted[this.url] = true;
+  }
+
   this.wrapper = '#' + element_settings.wrapper;
 
   // If there isn't a form, jQuery.ajax() will be used instead, allowing us to
@@ -142,7 +149,7 @@ Drupal.ajax = function (base, element, element_settings) {
   // The 'this' variable will not persist inside of the options object.
   var ajax = this;
   ajax.options = {
-    url: ajax.url,
+    url: Drupal.sanitizeAjaxUrl(ajax.url),
     data: ajax.submit,
     beforeSerialize: function (element_settings, options) {
       return ajax.beforeSerialize(element_settings, options);
@@ -155,26 +162,67 @@ Drupal.ajax = function (base, element, element_settings) {
       ajax.ajaxing = true;
       return ajax.beforeSend(xmlhttprequest, options);
     },
-    success: function (response, status) {
+    success: function (response, status, xmlhttprequest) {
       // Sanity check for browser support (object expected).
       // When using iFrame uploads, responses must be returned as a string.
       if (typeof response == 'string') {
         response = $.parseJSON(response);
       }
+
+      // Prior to invoking the response's commands, verify that they can be
+      // trusted by checking for a response header. See
+      // ajax_set_verification_header() for details.
+      // - Empty responses are harmless so can bypass verification. This avoids
+      //   an alert message for server-generated no-op responses that skip Ajax
+      //   rendering.
+      // - Ajax objects with trusted URLs (e.g., ones defined server-side via
+      //   #ajax) can bypass header verification. This is especially useful for
+      //   Ajax with multipart forms. Because IFRAME transport is used, the
+      //   response headers cannot be accessed for verification.
+      if (response !== null && !Drupal.settings.urlIsAjaxTrusted[ajax.url]) {
+        if (xmlhttprequest.getResponseHeader('X-Drupal-Ajax-Token') !== '1') {
+          var customMessage = Drupal.t("The response failed verification so will not be processed.");
+          return ajax.error(xmlhttprequest, ajax.url, customMessage);
+        }
+      }
+
       return ajax.success(response, status);
     },
-    complete: function (response, status) {
+    complete: function (xmlhttprequest, status) {
       ajax.ajaxing = false;
       if (status == 'error' || status == 'parsererror') {
-        return ajax.error(response, ajax.url);
+        return ajax.error(xmlhttprequest, ajax.url);
       }
     },
     dataType: 'json',
+    jsonp: false,
     type: 'POST'
   };
 
+  // For multipart forms (e.g., file uploads), jQuery Form targets the form
+  // submission to an iframe instead of using an XHR object. The initial "src"
+  // of the iframe, prior to the form submission, is set to options.iframeSrc.
+  // "about:blank" is the semantically correct, standards-compliant, way to
+  // initialize a blank iframe; however, some old IE versions (possibly only 6)
+  // incorrectly report a mixed content warning when iframes with an
+  // "about:blank" src are added to a parent document with an https:// origin.
+  // jQuery Form works around this by defaulting to "javascript:false" instead,
+  // but that breaks on Chrome 83, so here we force the semantically correct
+  // behavior for all browsers except old IE.
+  // @see https://www.drupal.org/project/drupal/issues/3143016
+  // @see https://github.com/jquery-form/form/blob/df9cb101b9c9c085c8d75ad980c7ff1cf62063a1/jquery.form.js#L68
+  // @see https://bugs.chromium.org/p/chromium/issues/detail?id=1084874
+  // @see https://html.spec.whatwg.org/multipage/browsers.html#creating-browsing-contexts
+  // @see https://developer.mozilla.org/en-US/docs/Web/Security/Same-origin_policy
+  if (navigator.userAgent.indexOf("MSIE") === -1) {
+    ajax.options.iframeSrc = 'about:blank';
+  }
+
   // Bind the ajaxSubmit function to the element event.
   $(ajax.element).bind(element_settings.event, function (event) {
+    if (!Drupal.settings.urlIsAjaxTrusted[ajax.url] && !Drupal.urlIsLocal(ajax.url)) {
+      throw new Error(Drupal.t('The callback URL is not local and not trusted: !url', {'!url': ajax.url}));
+    }
     return ajax.eventResponse(this, event);
   });
 
@@ -348,7 +396,7 @@ Drupal.ajax.prototype.beforeSend = function (xmlhttprequest, options) {
     // this is only needed for IFRAME submissions.
     var v = $.fieldValue(this.element);
     if (v !== null) {
-      options.extraData[this.element.name] = v;
+      options.extraData[this.element.name] = Drupal.checkPlain(v);
     }
   }
 
@@ -447,8 +495,8 @@ Drupal.ajax.prototype.getEffect = function (response) {
 /**
  * Handler for the form redirection error.
  */
-Drupal.ajax.prototype.error = function (response, uri) {
-  alert(Drupal.ajaxError(response, uri));
+Drupal.ajax.prototype.error = function (xmlhttprequest, uri, customMessage) {
+  Drupal.displayAjaxError(Drupal.ajaxError(xmlhttprequest, uri, customMessage));
   // Remove the progress element.
   if (this.progress.element) {
     $(this.progress.element).remove();
@@ -462,7 +510,7 @@ Drupal.ajax.prototype.error = function (response, uri) {
   $(this.element).removeClass('progress-disabled').removeAttr('disabled');
   // Reattach behaviors, if they were detached in beforeSerialize().
   if (this.form) {
-    var settings = response.settings || this.settings || Drupal.settings;
+    var settings = this.settings || Drupal.settings;
     Drupal.attachBehaviors(this.form, settings);
   }
 };
@@ -616,6 +664,33 @@ Drupal.ajax.prototype.commands = {
       .removeClass('odd even')
       .filter(':even').addClass('odd').end()
       .filter(':odd').addClass('even');
+  },
+
+  /**
+   * Command to add css.
+   *
+   * Uses the proprietary addImport method if available as browsers which
+   * support that method ignore @import statements in dynamically added
+   * stylesheets.
+   */
+  add_css: function (ajax, response, status) {
+    // Add the styles in the normal way.
+    $('head').prepend(response.data);
+    // Add imports in the styles using the addImport method if available.
+    var match, importMatch = /^@import url\("(.*)"\);$/igm;
+    if (document.styleSheets[0].addImport && importMatch.test(response.data)) {
+      importMatch.lastIndex = 0;
+      while (match = importMatch.exec(response.data)) {
+        document.styleSheets[0].addImport(match[1]);
+      }
+    }
+  },
+
+  /**
+   * Command to update a form's build ID.
+   */
+  updateBuildId: function(ajax, response, status) {
+    $('input[name="form_build_id"][value="' + response['old'] + '"]').val(response['new']);
   }
 };
 

misc/autocomplete.js

@@ -114,6 +114,7 @@ Drupal.jsAC.prototype.onkeyup = function (input, e) {
  */
 Drupal.jsAC.prototype.select = function (node) {
   this.input.value = $(node).data('autocompleteValue');
+  $(this.input).trigger('autocompleteSelect', [node]);
 };
 
 /**
@@ -167,7 +168,7 @@ Drupal.jsAC.prototype.unhighlight = function (node) {
 Drupal.jsAC.prototype.hidePopup = function (keycode) {
   // Select item if the right key or mousebutton was pressed.
   if (this.selected && ((keycode && keycode != 46 && keycode != 8 && keycode != 27) || !keycode)) {
-    this.input.value = $(this.selected).data('autocompleteValue');
+    this.select(this.selected);
   }
   // Hide popup.
   var popup = this.popup;
@@ -220,7 +221,7 @@ Drupal.jsAC.prototype.found = function (matches) {
   for (key in matches) {
     $('<li></li>')
       .html($('<div></div>').html(matches[key]))
-      .mousedown(function () { ac.select(this); })
+      .mousedown(function () { ac.hidePopup(this); })
       .mouseover(function () { ac.highlight(this); })
       .mouseout(function () { ac.unhighlight(this); })
       .data('autocompleteValue', key)
@@ -270,8 +271,11 @@ Drupal.ACDB.prototype.search = function (searchString) {
   var db = this;
   this.searchString = searchString;
 
-  // See if this string needs to be searched for anyway.
-  searchString = searchString.replace(/^\s+|\s+$/, '');
+  // See if this string needs to be searched for anyway. The pattern ../ is
+  // stripped since it may be misinterpreted by the browser.
+  searchString = searchString.replace(/^\s+|\.{2,}\/|\s+$/g, '');
+  // Skip empty search strings, or search strings ending with a comma, since
+  // that is the separator between search terms.
   if (searchString.length <= 0 ||
     searchString.charAt(searchString.length - 1) == ',') {
     return;
@@ -293,8 +297,9 @@ Drupal.ACDB.prototype.search = function (searchString) {
     // encodeURIComponent to allow autocomplete search terms to contain slashes.
     $.ajax({
       type: 'GET',
-      url: db.uri + '/' + Drupal.encodePath(searchString),
+      url: Drupal.sanitizeAjaxUrl(db.uri + '/' + Drupal.encodePath(searchString)),
       dataType: 'json',
+      jsonp: false,
       success: function (matches) {
         if (typeof matches.status == 'undefined' || matches.status != 0) {
           db.cache[searchString] = matches;
@@ -306,7 +311,7 @@ Drupal.ACDB.prototype.search = function (searchString) {
         }
       },
       error: function (xmlhttp) {
-        alert(Drupal.ajaxError(xmlhttp, db.uri));
+        Drupal.displayAjaxError(Drupal.ajaxError(xmlhttp, db.uri));
       }
     });
   }, this.delay);

misc/brumann/polyfill-unserialize/.gitignore

@@ -0,0 +1,4 @@
+/vendor/
+/phpunit.xml
+/.composer.lock
+

misc/brumann/polyfill-unserialize/.travis.yml

@@ -0,0 +1,20 @@
+language: php
+
+sudo: false
+
+php:
+  - '5.3'
+  - '5.4'
+  - '5.5'
+  - '5.6'
+  - '7.0'
+  - '7.1'
+
+before_install:
+  - phpenv config-rm xdebug.ini
+  - composer self-update
+
+install:
+  - composer install
+
+script: phpunit

misc/brumann/polyfill-unserialize/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016 Denis Brumann
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

misc/brumann/polyfill-unserialize/README.md

@@ -0,0 +1,61 @@
+Polyfill unserialize [![Build Status](https://travis-ci.org/dbrumann/polyfill-unserialize.svg?branch=master)](https://travis-ci.org/dbrumann/polyfill-unserialize)
+===
+
+Backports unserialize options introduced in PHP 7.0 to older PHP versions.
+This was originally designed as a Proof of Concept for Symfony Issue [#21090](https://github.com/symfony/symfony/pull/21090).
+
+You can use this package in projects that rely on PHP versions older than PHP 7.0.
+In case you are using PHP 7.0+ the original `unserialize()` will be used instead.
+
+From the [documentation](https://secure.php.net/manual/en/function.unserialize.php):
+
+> Warning: Do not pass untrusted user input to unserialize(). Unserialization can
+> result in code being loaded and executed due to object instantiation
+> and autoloading, and a malicious user may be able to exploit this.
+
+This warning holds true even when `allowed_classes` is used.
+
+Requirements
+------------
+
+ - PHP 5.3+
+
+Installation
+------------
+
+You can install this package via composer:
+
+```
+composer require brumann/polyfill-unserialize "^1.0"
+```
+
+Known Issues
+------------
+
+There is a mismatch in behavior when `allowed_classes` in `$options` is not
+of the correct type (array or boolean). PHP 7.1 will issue a warning, whereas
+PHP 7.0 will not. I opted to copy the behavior of the former.
+
+Tests
+-----
+
+You can run the test suite using PHPUnit. It is intentionally not bundled as
+dev dependency to make sure this package has the lowest restrictions on the
+implementing system as possible.
+
+Please read the [PHPUnit Manual](https://phpunit.de/manual/current/en/installation.html)
+for information how to install it on your system.
+
+You can run the test suite as follows:
+
+```
+phpunit -c phpunit.xml.dist tests/
+```
+
+Contributing
+------------
+
+This package is considered feature complete. As such I will likely not update it
+unless there are security issues.
+
+Should you find any bugs or have questions, feel free to submit an Issue or a Pull Request.

misc/brumann/polyfill-unserialize/composer.json

@@ -0,0 +1,26 @@
+{
+    "name": "brumann/polyfill-unserialize",
+    "description": "Backports unserialize options introduced in PHP 7.0 to older PHP versions.",
+    "type": "library",
+    "license": "MIT",
+    "authors": [
+        {
+            "name": "Denis Brumann",
+            "email": "denis.brumann@sensiolabs.de"
+        }
+    ],
+    "autoload": {
+        "psr-4": {
+            "Brumann\\Polyfill\\": "src/"
+        }
+    },
+    "autoload-dev": {
+        "psr-4": {
+            "Tests\\Brumann\\Polyfill\\": "tests/"
+        }
+    },
+    "minimum-stability": "stable",
+    "require": {
+        "php": "^5.3|^7.0"
+    }
+}

misc/brumann/polyfill-unserialize/phpunit.xml.dist

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<phpunit
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:noNamespaceSchemaLocation="http://schema.phpunit.de/4.1/phpunit.xsd"
+    backupGlobals="false"
+    colors="true"
+    bootstrap="vendor/autoload.php"
+>
+    <php>
+        <ini name="error_reporting" value="-1" />
+    </php>
+
+    <testsuites>
+        <testsuite name="Brumann\Polyfill Test Suite">
+            <directory>./tests/</directory>
+        </testsuite>
+    </testsuites>
+
+    <filter>
+        <whitelist>
+            <directory>./src/</directory>
+        </whitelist>
+    </filter>
+</phpunit>

misc/brumann/polyfill-unserialize/src/Unserialize.php

@@ -0,0 +1,58 @@
+<?php
+
+namespace Brumann\Polyfill;
+
+final class Unserialize
+{
+    /**
+     * @see https://secure.php.net/manual/en/function.unserialize.php
+     *
+     * @param string $serialized Serialized data
+     * @param array $options Associative array containing options
+     *
+     * @return mixed
+     */
+    public static function unserialize($serialized, array $options = array())
+    {
+        if (PHP_VERSION_ID >= 70000) {
+            return \unserialize($serialized, $options);
+        }
+        if (!array_key_exists('allowed_classes', $options)) {
+            $options['allowed_classes'] = true;
+        }
+        $allowedClasses = $options['allowed_classes'];
+        if (true === $allowedClasses) {
+            return \unserialize($serialized);
+        }
+        if (false === $allowedClasses) {
+            $allowedClasses = array();
+        }
+        if (!is_array($allowedClasses)) {
+            trigger_error(
+                'unserialize(): allowed_classes option should be array or boolean',
+                E_USER_WARNING
+            );
+            $allowedClasses = array();
+        }
+
+        $sanitizedSerialized = preg_replace_callback(
+            '/(^|;)O:\d+:"([^"]*)":(\d+):{/',
+            function ($match) use ($allowedClasses) {
+                list($completeMatch, $leftBorder, $className, $objectSize) = $match;
+                if (in_array($className, $allowedClasses)) {
+                    return $completeMatch;
+                } else {
+                    return sprintf(
+                        '%sO:22:"__PHP_Incomplete_Class":%d:{s:27:"__PHP_Incomplete_Class_Name";%s',
+                        $leftBorder,
+                        $objectSize + 1, // size of object + 1 for added string
+                        \serialize($className)
+                    );
+                }
+            },
+            $serialized
+        );
+
+        return \unserialize($sanitizedSerialized);
+    }
+}

misc/drupal.js

@@ -27,6 +27,42 @@ $.fn.init = function (selector, context, rootjQuery) {
 };
 $.fn.init.prototype = jquery_init.prototype;
 
+/**
+ * Pre-filter Ajax requests to guard against XSS attacks.
+ *
+ * See https://github.com/jquery/jquery/issues/2432
+ */
+if ($.ajaxPrefilter) {
+  // For newer versions of jQuery, use an Ajax prefilter to prevent
+  // auto-executing script tags from untrusted domains. This is similar to the
+  // fix that is built in to jQuery 3.0 and higher.
+  $.ajaxPrefilter(function (s) {
+    if (s.crossDomain) {
+      s.contents.script = false;
+    }
+  });
+}
+else if ($.httpData) {
+  // For the version of jQuery that ships with Drupal core, override
+  // jQuery.httpData to prevent auto-detecting "script" data types from
+  // untrusted domains.
+  var jquery_httpData = $.httpData;
+  $.httpData = function (xhr, type, s) {
+    // @todo Consider backporting code from newer jQuery versions to check for
+    //   a cross-domain request here, rather than using Drupal.urlIsLocal() to
+    //   block scripts from all URLs that are not on the same site.
+    if (!type && !Drupal.urlIsLocal(s.url)) {
+      var content_type = xhr.getResponseHeader('content-type') || '';
+      if (content_type.indexOf('javascript') >= 0) {
+        // Default to a safe data type.
+        type = 'text';
+      }
+    }
+    return jquery_httpData.call(this, xhr, type, s);
+  };
+  $.httpData.prototype = jquery_httpData.prototype;
+}
+
 /**
  * Attach all registered behaviors to a page element.
  *
@@ -137,7 +173,7 @@ Drupal.detachBehaviors = function (context, settings, trigger) {
  */
 Drupal.checkPlain = function (str) {
   var character, regex,
-      replace = { '&': '&amp;', '"': '&quot;', '<': '&lt;', '>': '&gt;' };
+      replace = { '&': '&amp;', "'": '&#39;', '"': '&quot;', '<': '&lt;', '>': '&gt;' };
   str = String(str);
   for (character in replace) {
     if (replace.hasOwnProperty(character)) {
@@ -168,23 +204,76 @@ Drupal.checkPlain = function (str) {
 Drupal.formatString = function(str, args) {
   // Transform arguments before inserting them.
   for (var key in args) {
-    switch (key.charAt(0)) {
-      // Escaped only.
-      case '@':
-        args[key] = Drupal.checkPlain(args[key]);
-      break;
-      // Pass-through.
-      case '!':
-        break;
-      // Escaped and placeholder.
-      case '%':
-      default:
-        args[key] = Drupal.theme('placeholder', args[key]);
-        break;
+    if (args.hasOwnProperty(key)) {
+      switch (key.charAt(0)) {
+        // Escaped only.
+        case '@':
+          args[key] = Drupal.checkPlain(args[key]);
+          break;
+        // Pass-through.
+        case '!':
+          break;
+        // Escaped and placeholder.
+        default:
+          args[key] = Drupal.theme('placeholder', args[key]);
+          break;
+      }
     }
-    str = str.replace(key, args[key]);
   }
-  return str;
+
+  return Drupal.stringReplace(str, args, null);
+};
+
+/**
+ * Replace substring.
+ *
+ * The longest keys will be tried first. Once a substring has been replaced,
+ * its new value will not be searched again.
+ *
+ * @param {String} str
+ *   A string with placeholders.
+ * @param {Object} args
+ *   Key-value pairs.
+ * @param {Array|null} keys
+ *   Array of keys from the "args".  Internal use only.
+ *
+ * @return {String}
+ *   Returns the replaced string.
+ */
+Drupal.stringReplace = function (str, args, keys) {
+  if (str.length === 0) {
+    return str;
+  }
+
+  // If the array of keys is not passed then collect the keys from the args.
+  if (!$.isArray(keys)) {
+    keys = [];
+    for (var k in args) {
+      if (args.hasOwnProperty(k)) {
+        keys.push(k);
+      }
+    }
+
+    // Order the keys by the character length. The shortest one is the first.
+    keys.sort(function (a, b) { return a.length - b.length; });
+  }
+
+  if (keys.length === 0) {
+    return str;
+  }
+
+  // Take next longest one from the end.
+  var key = keys.pop();
+  var fragments = str.split(key);
+
+  if (keys.length) {
+    for (var i = 0; i < fragments.length; i++) {
+      // Process each fragment with a copy of remaining keys.
+      fragments[i] = Drupal.stringReplace(fragments[i], args, keys.slice(0));
+    }
+  }
+
+  return fragments.join(args[key]);
 };
 
 /**
@@ -251,7 +340,7 @@ Drupal.t = function (str, args, options) {
  *   A translated string.
  */
 Drupal.formatPlural = function (count, singular, plural, args, options) {
-  var args = args || {};
+  args = args || {};
   args['@count'] = count;
   // Determine the index of the plural form.
   var index = Drupal.locale.pluralFormula ? Drupal.locale.pluralFormula(args['@count']) : ((args['@count'] == 1) ? 0 : 1);
@@ -269,6 +358,89 @@ Drupal.formatPlural = function (count, singular, plural, args, options) {
   }
 };
 
+/**
+ * Returns the passed in URL as an absolute URL.
+ *
+ * @param url
+ *   The URL string to be normalized to an absolute URL.
+ *
+ * @return
+ *   The normalized, absolute URL.
+ *
+ * @see https://github.com/angular/angular.js/blob/v1.4.4/src/ng/urlUtils.js
+ * @see https://grack.com/blog/2009/11/17/absolutizing-url-in-javascript
+ * @see https://github.com/jquery/jquery-ui/blob/1.11.4/ui/tabs.js#L53
+ */
+Drupal.absoluteUrl = function (url) {
+  var urlParsingNode = document.createElement('a');
+
+  // Decode the URL first; this is required by IE <= 6. Decoding non-UTF-8
+  // strings may throw an exception.
+  try {
+    url = decodeURIComponent(url);
+  } catch (e) {}
+
+  urlParsingNode.setAttribute('href', url);
+
+  // IE <= 7 normalizes the URL when assigned to the anchor node similar to
+  // the other browsers.
+  return urlParsingNode.cloneNode(false).href;
+};
+
+/**
+ * Returns true if the URL is within Drupal's base path.
+ *
+ * @param url
+ *   The URL string to be tested.
+ *
+ * @return
+ *   Boolean true if local.
+ *
+ * @see https://github.com/jquery/jquery-ui/blob/1.11.4/ui/tabs.js#L58
+ */
+Drupal.urlIsLocal = function (url) {
+  // Always use browser-derived absolute URLs in the comparison, to avoid
+  // attempts to break out of the base path using directory traversal.
+  var absoluteUrl = Drupal.absoluteUrl(url);
+  var protocol = location.protocol;
+
+  // Consider URLs that match this site's base URL but use HTTPS instead of HTTP
+  // as local as well.
+  if (protocol === 'http:' && absoluteUrl.indexOf('https:') === 0) {
+    protocol = 'https:';
+  }
+  var baseUrl = protocol + '//' + location.host + Drupal.settings.basePath.slice(0, -1);
+
+  // Decoding non-UTF-8 strings may throw an exception.
+  try {
+    absoluteUrl = decodeURIComponent(absoluteUrl);
+  } catch (e) {}
+  try {
+    baseUrl = decodeURIComponent(baseUrl);
+  } catch (e) {}
+
+  // The given URL matches the site's base URL, or has a path under the site's
+  // base URL.
+  return absoluteUrl === baseUrl || absoluteUrl.indexOf(baseUrl + '/') === 0;
+};
+
+/**
+ * Sanitizes a URL for use with jQuery.ajax().
+ *
+ * @param url
+ *   The URL string to be sanitized.
+ *
+ * @return
+ *   The sanitized URL.
+ */
+Drupal.sanitizeAjaxUrl = function (url) {
+  var regex = /\=\?(&|$)/;
+  while (url.match(regex)) {
+    url = url.replace(regex, '');
+  }
+  return url;
+}
+
 /**
  * Generate the themed representation of a Drupal object.
  *
@@ -347,10 +519,33 @@ Drupal.getSelection = function (element) {
   return { 'start': element.selectionStart, 'end': element.selectionEnd };
 };
 
+/**
+ * Add a global variable which determines if the window is being unloaded.
+ *
+ * This is primarily used by Drupal.displayAjaxError().
+ */
+Drupal.beforeUnloadCalled = false;
+$(window).bind('beforeunload pagehide', function () {
+    Drupal.beforeUnloadCalled = true;
+});
+
+/**
+ * Displays a JavaScript error from an Ajax response when appropriate to do so.
+ */
+Drupal.displayAjaxError = function (message) {
+  // Skip displaying the message if the user deliberately aborted (for example,
+  // by reloading the page or navigating to a different page) while the Ajax
+  // request was still ongoing. See, for example, the discussion at
+  // http://stackoverflow.com/questions/699941/handle-ajax-error-when-a-user-clicks-refresh.
+  if (!Drupal.beforeUnloadCalled) {
+    alert(message);
+  }
+};
+
 /**
  * Build an error message from an Ajax response.
  */
-Drupal.ajaxError = function (xmlhttp, uri) {
+Drupal.ajaxError = function (xmlhttp, uri, customMessage) {
   var statusCode, statusText, pathText, responseText, readyStateText, message;
   if (xmlhttp.status) {
     statusCode = "\n" + Drupal.t("An AJAX HTTP error occurred.") +  "\n" + Drupal.t("HTTP Result Code: !status", {'!status': xmlhttp.status});
@@ -383,7 +578,10 @@ Drupal.ajaxError = function (xmlhttp, uri) {
   // We don't need readyState except for status == 0.
   readyStateText = xmlhttp.status == 0 ? ("\n" + Drupal.t("ReadyState: !readyState", {'!readyState': xmlhttp.readyState})) : "";
 
-  message = statusCode + pathText + statusText + responseText + readyStateText;
+  // Additional message beyond what the xmlhttp object provides.
+  customMessage = customMessage ? ("\n" + Drupal.t("CustomMessage: !customMessage", {'!customMessage': customMessage})) : "";
+
+  message = statusCode + pathText + statusText + customMessage + responseText + readyStateText;
   return message;
 };
 

misc/jquery-extend-3.4.0.js

@@ -0,0 +1,112 @@
+/**
+ * For jQuery versions less than 3.4.0, this replaces the jQuery.extend
+ * function with the one from jQuery 3.4.0, slightly modified (documented
+ * below) to be compatible with older jQuery versions and browsers.
+ *
+ * This provides the Object.prototype pollution vulnerability fix to Drupal
+ * installations running older jQuery versions, including the versions shipped
+ * with Drupal core and https://www.drupal.org/project/jquery_update.
+ *
+ * @see https://github.com/jquery/jquery/pull/4333
+ */
+
+(function (jQuery) {
+
+// Do not override jQuery.extend() if the jQuery version is already >=3.4.0.
+var versionParts = jQuery.fn.jquery.split('.');
+var majorVersion = parseInt(versionParts[0]);
+var minorVersion = parseInt(versionParts[1]);
+var patchVersion = parseInt(versionParts[2]);
+var isPreReleaseVersion = (patchVersion.toString() !== versionParts[2]);
+if (
+  (majorVersion > 3) ||
+  (majorVersion === 3 && minorVersion > 4) ||
+  (majorVersion === 3 && minorVersion === 4 && patchVersion > 0) ||
+  (majorVersion === 3 && minorVersion === 4 && patchVersion === 0 && !isPreReleaseVersion)
+) {
+  return;
+}
+
+/**
+ * This is almost verbatim copied from jQuery 3.4.0.
+ *
+ * Only two minor changes have been made:
+ * - The call to isFunction() is changed to jQuery.isFunction().
+ * - The two calls to Array.isArray() is changed to jQuery.isArray().
+ *
+ * The above two changes ensure compatibility with all older jQuery versions
+ * (1.4.4 - 3.3.1) and older browser versions (e.g., IE8).
+ */
+jQuery.extend = jQuery.fn.extend = function() {
+  var options, name, src, copy, copyIsArray, clone,
+    target = arguments[ 0 ] || {},
+    i = 1,
+    length = arguments.length,
+    deep = false;
+
+  // Handle a deep copy situation
+  if ( typeof target === "boolean" ) {
+    deep = target;
+
+    // Skip the boolean and the target
+    target = arguments[ i ] || {};
+    i++;
+  }
+
+  // Handle case when target is a string or something (possible in deep copy)
+  if ( typeof target !== "object" && !jQuery.isFunction( target ) ) {
+    target = {};
+  }
+
+  // Extend jQuery itself if only one argument is passed
+  if ( i === length ) {
+    target = this;
+    i--;
+  }
+
+  for ( ; i < length; i++ ) {
+
+    // Only deal with non-null/undefined values
+    if ( ( options = arguments[ i ] ) != null ) {
+
+      // Extend the base object
+      for ( name in options ) {
+        copy = options[ name ];
+
+        // Prevent Object.prototype pollution
+        // Prevent never-ending loop
+        if ( name === "__proto__" || target === copy ) {
+          continue;
+        }
+
+        // Recurse if we're merging plain objects or arrays
+        if ( deep && copy && ( jQuery.isPlainObject( copy ) ||
+          ( copyIsArray = jQuery.isArray( copy ) ) ) ) {
+          src = target[ name ];
+
+          // Ensure proper type for the source value
+          if ( copyIsArray && !jQuery.isArray( src ) ) {
+            clone = [];
+          } else if ( !copyIsArray && !jQuery.isPlainObject( src ) ) {
+            clone = {};
+          } else {
+            clone = src;
+          }
+          copyIsArray = false;
+
+          // Never move original objects, clone them
+          target[ name ] = jQuery.extend( deep, clone, copy );
+
+          // Don't bring in undefined values
+        } else if ( copy !== undefined ) {
+          target[ name ] = copy;
+        }
+      }
+    }
+  }
+
+  // Return the modified object
+  return target;
+};
+
+})(jQuery);

misc/jquery-html-prefilter-3.5.0-backport.js

@@ -0,0 +1,251 @@
+/**
+ * For jQuery versions less than 3.5.0, this replaces the jQuery.htmlPrefilter()
+ * function with one that fixes these security vulnerabilities while also
+ * retaining the pre-3.5.0 behavior where it's safe to do so.
+ * - https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-11022
+ * - https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-11023
+ *
+ * Additionally, for jQuery versions that do not have a jQuery.htmlPrefilter()
+ * function (1.x prior to 1.12 and 2.x prior to 2.2), this adds it, and
+ * extends the functions that need to call it to do so.
+ *
+ * Drupal core's jQuery version is 1.4.4, but jQuery Update can provide a
+ * different version, so this covers all versions between 1.4.4 and 3.4.1.
+ * The GitHub links in the code comments below link to jQuery 1.5 code, because
+ * 1.4.4 isn't on GitHub, but the referenced code didn't change from 1.4.4 to
+ * 1.5.
+ */
+
+(function (jQuery) {
+
+  // Parts of this backport differ by jQuery version.
+  var versionParts = jQuery.fn.jquery.split('.');
+  var majorVersion = parseInt(versionParts[0]);
+  var minorVersion = parseInt(versionParts[1]);
+
+  // No backport is needed if we're already on jQuery 3.5 or higher.
+  if ( (majorVersion > 3) || (majorVersion === 3 && minorVersion >= 5) ) {
+    return;
+  }
+
+  // Prior to jQuery 3.5, jQuery converted XHTML-style self-closing tags to
+  // their XML equivalent: e.g., "<div />" to "<div></div>". This is
+  // problematic for several reasons, including that it's vulnerable to XSS
+  // attacks. However, since this was jQuery's behavior for many years, many
+  // Drupal modules and jQuery plugins may be relying on it. Therefore, we
+  // preserve that behavior, but for a limited set of tags only, that we believe
+  // to not be vulnerable. This is the set of HTML tags that satisfy all of the
+  // following conditions:
+  // - In DOMPurify's list of HTML tags. If an HTML tag isn't safe enough to
+  //   appear in that list, then we don't want to mess with it here either.
+  //   @see https://github.com/cure53/DOMPurify/blob/2.0.11/dist/purify.js#L128
+  // - A normal element (not a void, template, text, or foreign element).
+  //   @see https://html.spec.whatwg.org/multipage/syntax.html#elements-2
+  // - An element that is still defined by the current HTML specification
+  //   (not a deprecated element), because we do not want to rely on how
+  //   browsers parse deprecated elements.
+  //   @see https://developer.mozilla.org/en-US/docs/Web/HTML/Element
+  // - Not 'html', 'head', or 'body', because this pseudo-XHTML expansion is
+  //   designed for fragments, not entire documents.
+  // - Not 'colgroup', because due to an idiosyncrasy of jQuery's original
+  //   regular expression, it didn't match on colgroup, and we don't want to
+  //   introduce a behavior change for that.
+  var selfClosingTagsToReplace = [
+    'a', 'abbr', 'address', 'article', 'aside', 'audio', 'b', 'bdi', 'bdo',
+    'blockquote', 'button', 'canvas', 'caption', 'cite', 'code', 'data',
+    'datalist', 'dd', 'del', 'details', 'dfn', 'div', 'dl', 'dt', 'em',
+    'fieldset', 'figcaption', 'figure', 'footer', 'form', 'h1', 'h2', 'h3',
+    'h4', 'h5', 'h6', 'header', 'hgroup', 'i', 'ins', 'kbd', 'label', 'legend',
+    'li', 'main', 'map', 'mark', 'menu', 'meter', 'nav', 'ol', 'optgroup',
+    'option', 'output', 'p', 'picture', 'pre', 'progress', 'q', 'rp', 'rt',
+    'ruby', 's', 'samp', 'section', 'select', 'small', 'source', 'span',
+    'strong', 'sub', 'summary', 'sup', 'table', 'tbody', 'td', 'tfoot', 'th',
+    'thead', 'time', 'tr', 'u', 'ul', 'var', 'video'
+  ];
+
+  // Define regular expressions for <TAG/> and <TAG ATTRIBUTES/>. Doing this as
+  // two expressions makes it easier to target <a/> without also targeting
+  // every tag that starts with "a".
+  var xhtmlRegExpGroup = '(' + selfClosingTagsToReplace.join('|') + ')';
+  var whitespace = '[\\x20\\t\\r\\n\\f]';
+  var rxhtmlTagWithoutSpaceOrAttributes = new RegExp('<' + xhtmlRegExpGroup + '\\/>', 'gi');
+  var rxhtmlTagWithSpaceAndMaybeAttributes = new RegExp('<' + xhtmlRegExpGroup + '(' + whitespace + '[^>]*)\\/>', 'gi');
+
+  // jQuery 3.5 also fixed a vulnerability for when </select> appears within
+  // an <option> or <optgroup>, but it did that in local code that we can't
+  // backport directly. Instead, we filter such cases out. To do so, we need to
+  // determine when jQuery would otherwise invoke the vulnerable code, which it
+  // uses this regular expression to determine. The regular expression changed
+  // for version 3.0.0 and changed again for 3.4.0.
+  // @see https://github.com/jquery/jquery/blob/1.5/jquery.js#L4958
+  // @see https://github.com/jquery/jquery/blob/3.0.0/dist/jquery.js#L4584
+  // @see https://github.com/jquery/jquery/blob/3.4.0/dist/jquery.js#L4712
+  var rtagName;
+  if (majorVersion < 3) {
+    rtagName = /<([\w:]+)/;
+  }
+  else if (minorVersion < 4) {
+    rtagName = /<([a-z][^\/\0>\x20\t\r\n\f]+)/i;
+  }
+  else {
+    rtagName = /<([a-z][^\/\0>\x20\t\r\n\f]*)/i;
+  }
+
+  // The regular expression that jQuery uses to determine which self-closing
+  // tags to expand to open and close tags. This is vulnerable, because it
+  // matches all tag names except the few excluded ones. We only use this
+  // expression for determining vulnerability. The expression changed for
+  // version 3, but we only need to check for vulnerability in versions 1 and 2,
+  // so we use the expression from those versions.
+  // @see https://github.com/jquery/jquery/blob/1.5/jquery.js#L4957
+  var rxhtmlTag = /<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/gi;
+
+  jQuery.extend({
+    htmlPrefilter: function (html) {
+      // This is how jQuery determines the first tag in the HTML.
+      // @see https://github.com/jquery/jquery/blob/1.5/jquery.js#L5521
+      var tag = ( rtagName.exec( html ) || [ "", "" ] )[ 1 ].toLowerCase();
+
+      // It is not valid HTML for <option> or <optgroup> to have <select> as
+      // either a descendant or sibling, and attempts to inject one can cause
+      // XSS on jQuery versions before 3.5. Since this is invalid HTML and a
+      // possible XSS attack, reject the entire string.
+      // @see https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-11023
+      if ((tag === 'option' || tag === 'optgroup') && html.match(/<\/?select/i)) {
+        html = '';
+      }
+
+      // Retain jQuery's prior to 3.5 conversion of pseudo-XHTML, but for only
+      // the tags in the `selfClosingTagsToReplace` list defined above.
+      // @see https://github.com/jquery/jquery/blob/1.5/jquery.js#L5518
+      // @see https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-11022
+      html = html.replace(rxhtmlTagWithoutSpaceOrAttributes, "<$1></$1>");
+      html = html.replace(rxhtmlTagWithSpaceAndMaybeAttributes, "<$1$2></$1>");
+
+      // Prior to jQuery 1.12 and 2.2, this function gets called (via code later
+      // in this file) in addition to, rather than instead of, the unsafe
+      // expansion of self-closing tags (including ones not in the list above).
+      // We can't prevent that unsafe expansion from running, so instead we
+      // check to make sure that it doesn't affect the DOM returned by the
+      // browser's parsing logic. If it does affect it, then it's vulnerable to
+      // XSS, so we reject the entire string.
+      if ( (majorVersion === 1 && minorVersion < 12) || (majorVersion === 2 && minorVersion < 2) ) {
+        var htmlRisky = html.replace(rxhtmlTag, "<$1></$2>");
+        if (htmlRisky !== html) {
+          // Even though htmlRisky and html are different strings, they might
+          // represent the same HTML structure once parsed, in which case,
+          // htmlRisky is actually safe. We can ask the browser to parse both
+          // to find out, but the browser can't parse table fragments (e.g., a
+          // root-level "<td>"), so we need to wrap them. We just need this
+          // technique to work on all supported browsers; we don't need to
+          // copy from the specific jQuery version we're using.
+          // @see https://github.com/jquery/jquery/blob/3.5.1/dist/jquery.js#L4939
+          var wrapMap = {
+            thead: [ 1, "<table>", "</table>" ],
+            col: [ 2, "<table><colgroup>", "</colgroup></table>" ],
+            tr: [ 2, "<table><tbody>", "</tbody></table>" ],
+            td: [ 3, "<table><tbody><tr>", "</tr></tbody></table>" ],
+          };
+          wrapMap.tbody = wrapMap.tfoot = wrapMap.colgroup = wrapMap.caption = wrapMap.thead;
+          wrapMap.th = wrapMap.td;
+
+          // Function to wrap HTML into something that a browser can parse.
+          // @see https://github.com/jquery/jquery/blob/3.5.1/dist/jquery.js#L5032
+          var getWrappedHtml = function (html) {
+            var wrap = wrapMap[tag];
+            if (wrap) {
+              html = wrap[1] + html + wrap[2];
+            }
+            return html;
+          };
+
+          // Function to return canonical HTML after parsing it. This parses
+          // only; it doesn't execute scripts.
+          // @see https://github.com/jquery/jquery-migrate/blob/3.3.0/src/jquery/manipulation.js#L5
+          var getParsedHtml = function (html) {
+            var doc = window.document.implementation.createHTMLDocument( "" );
+            doc.body.innerHTML = html;
+            return doc.body ? doc.body.innerHTML : '';
+          };
+
+          // If the browser couldn't parse either one successfully, or if
+          // htmlRisky parses differently than html, then html is vulnerable,
+          // so reject it.
+          var htmlParsed = getParsedHtml(getWrappedHtml(html));
+          var htmlRiskyParsed = getParsedHtml(getWrappedHtml(htmlRisky));
+          if (htmlRiskyParsed === '' || htmlParsed === '' || (htmlRiskyParsed !== htmlParsed)) {
+            html = '';
+          }
+        }
+      }
+
+      return html;
+    }
+  });
+
+  // Prior to jQuery 1.12 and 2.2, jQuery.clean(), jQuery.buildFragment(), and
+  // jQuery.fn.html() did not call jQuery.htmlPrefilter(), so we add that.
+  if ( (majorVersion === 1 && minorVersion < 12) || (majorVersion === 2 && minorVersion < 2) ) {
+    // Filter the HTML coming into jQuery.fn.html().
+    var fnOriginalHtml = jQuery.fn.html;
+    jQuery.fn.extend({
+      // @see https://github.com/jquery/jquery/blob/1.5/jquery.js#L5147
+      html: function (value) {
+        if (typeof value === "string") {
+          value = jQuery.htmlPrefilter(value);
+        }
+        // .html() can be called as a setter (with an argument) or as a getter
+        // (without an argument), so invoke fnOriginalHtml() the same way that
+        // we were invoked.
+        return fnOriginalHtml.apply(this, arguments.length ? [value] : []);
+      }
+    });
+
+    // The regular expression that jQuery uses to determine if a string is HTML.
+    // Used by both clean() and buildFragment().
+    // @see https://github.com/jquery/jquery/blob/1.5/jquery.js#L4960
+    var rhtml = /<|&#?\w+;/;
+
+    // Filter HTML coming into:
+    // - jQuery.clean() for versions prior to 1.9.
+    // - jQuery.buildFragment() for 1.9 and above.
+    //
+    // The looping constructs in the two functions might be essentially
+    // identical, but they're each expressed here in the way that most closely
+    // matches their original expression in jQuery, so that we filter all of
+    // the items and only the items that jQuery will treat as HTML strings.
+    if (majorVersion === 1 && minorVersion < 9) {
+      var originalClean = jQuery.clean;
+      jQuery.extend({
+        // @see https://github.com/jquery/jquery/blob/1.5/jquery.js#L5493
+        'clean': function (elems, context, fragment, scripts) {
+          for ( var i = 0, elem; (elem = elems[i]) != null; i++ ) {
+            if ( typeof elem === "string" && rhtml.test( elem ) ) {
+              elems[i] = elem = jQuery.htmlPrefilter(elem);
+            }
+          }
+          return originalClean.call(this, elems, context, fragment, scripts);
+        }
+      });
+    }
+    else {
+      var originalBuildFragment = jQuery.buildFragment;
+      jQuery.extend({
+        // @see https://github.com/jquery/jquery/blob/1.9.0/jquery.js#L6419
+        'buildFragment': function (elems, context, scripts, selection) {
+          var l = elems.length;
+          for ( var i = 0; i < l; i++ ) {
+            var elem = elems[i];
+            if (elem || elem === 0) {
+              if ( jQuery.type( elem ) !== "object" && rhtml.test( elem ) ) {
+                elems[i] = elem = jQuery.htmlPrefilter(elem);
+              }
+            }
+          }
+          return originalBuildFragment.call(this, elems, context, scripts, selection);
+        }
+      });
+    }
+  }
+
+})(jQuery);

misc/states.js

@@ -373,7 +373,7 @@ states.Trigger.states = {
 
   checked: {
     'change': function () {
-      return this.attr('checked');
+      return this.is(':checked');
     }
   },
 
@@ -493,7 +493,11 @@ $(document).bind('state:disabled', function(e) {
 $(document).bind('state:required', function(e) {
   if (e.trigger) {
     if (e.value) {
-      $(e.target).closest('.form-item, .form-wrapper').find('label').append('<span class="form-required">*</span>');
+      var $label = $(e.target).closest('.form-item, .form-wrapper').find('label');
+      // Avoids duplicate required markers on initialization.
+      if (!$label.find('.form-required').length) {
+        $label.append('<span class="form-required">*</span>');
+      }
     }
     else {
       $(e.target).closest('.form-item, .form-wrapper').find('label .form-required').remove();

misc/tabledrag.js

@@ -106,8 +106,10 @@ Drupal.tableDrag = function (table, tableSettings) {
 
   // Add mouse bindings to the document. The self variable is passed along
   // as event handlers do not have direct access to the tableDrag object.
-  $(document).bind('mousemove', function (event) { return self.dragRow(event, self); });
-  $(document).bind('mouseup', function (event) { return self.dropRow(event, self); });
+  $(document).bind('mousemove pointermove', function (event) { return self.dragRow(event, self); });
+  $(document).bind('mouseup pointerup', function (event) { return self.dropRow(event, self); });
+  $(document).bind('touchmove', function (event) { return self.dragRow(event.originalEvent.touches[0], self); });
+  $(document).bind('touchend', function (event) { return self.dropRow(event.originalEvent.touches[0], self); });
 };
 
 /**
@@ -274,7 +276,10 @@ Drupal.tableDrag.prototype.makeDraggable = function (item) {
   });
 
   // Add the mousedown action for the handle.
-  handle.mousedown(function (event) {
+  handle.bind('mousedown touchstart pointerdown', function (event) {
+    if (event.originalEvent.type == "touchstart") {
+      event = event.originalEvent.touches[0];
+    }
     // Create a new dragObject recording the event information.
     self.dragObject = {};
     self.dragObject.initMouseOffset = self.getMouseOffset(item, event);
@@ -500,7 +505,7 @@ Drupal.tableDrag.prototype.dragRow = function (event, self) {
     if (self.indentEnabled) {
       var xDiff = self.currentMouseCoords.x - self.dragObject.indentMousePos.x;
       // Set the number of indentations the mouse has been moved left or right.
-      var indentDiff = Math.round(xDiff / self.indentAmount * self.rtl);
+      var indentDiff = Math.round(xDiff / self.indentAmount);
       // Indent the row with our estimated diff, which may be further
       // restricted according to the rows around this row.
       var indentChange = self.rowObject.indent(indentDiff);
@@ -575,13 +580,43 @@ Drupal.tableDrag.prototype.dropRow = function (event, self) {
  * Get the mouse coordinates from the event (allowing for browser differences).
  */
 Drupal.tableDrag.prototype.mouseCoords = function (event) {
-  if (event.pageX || event.pageY) {
-    return { x: event.pageX, y: event.pageY };
+
+  // Match both null and undefined, but not zero, by using != null.
+  // See https://stackoverflow.com/questions/2647867/how-to-determine-if-variable-is-undefined-or-null
+  if (event.pageX != null && event.pageY != null) {
+    return {x: event.pageX, y: event.pageY};
+  }
+
+  // Complete support for pointer events was only introduced to jQuery in
+  // version 1.11.1; between versions 1.7 and 1.11.0 pointer events have the
+  // pageX and pageY properties undefined. In those cases, the properties must
+  // be retrieved from the event.originalEvent object instead.
+  if (event.originalEvent && event.originalEvent.pageX != null && event.originalEvent.pageY != null) {
+    return {x: event.originalEvent.pageX, y: event.originalEvent.pageY};
+  }
+
+  // Some old browsers do not support MouseEvent.pageX and *.pageY at all.
+  // See https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent/pageY
+  // For those, we look at event.clientX and event.clientY instead.
+  if (event.clientX == null || event.clientY == null) {
+    // In some jQuery versions, some events created by jQuery do not have
+    // clientX and clientY. But the original event might have.
+    if (!event.originalEvent) {
+      throw new Error("The event has no coordinates, and no event.originalEvent.");
+    }
+    event = event.originalEvent;
+    if (event.clientX == null || event.clientY == null) {
+      throw new Error("The original event has no coordinates.");
+    }
   }
-  return {
-    x: event.clientX + document.body.scrollLeft - document.body.clientLeft,
-    y: event.clientY + document.body.scrollTop  - document.body.clientTop
-  };
+
+  // Copied from jQuery.event.fix() in jQuery 1.4.1.
+  // In newer jQuery versions, this code is in jQuery.event.mouseHooks.filter().
+  var doc = document.documentElement, body = document.body;
+  var pageX = event.clientX + ( doc && doc.scrollLeft || body && body.scrollLeft || 0 ) - ( doc && doc.clientLeft || body && body.clientLeft || 0 );
+  var pageY = event.clientY + ( doc && doc.scrollTop  || body && body.scrollTop  || 0 ) - ( doc && doc.clientTop  || body && body.clientTop  || 0 );
+
+  return {x: pageX, y: pageY};
 };
 
 /**

misc/tableselect.js

@@ -57,10 +57,14 @@ Drupal.tableSelect = function () {
     // Keep track of the last checked checkbox.
     lastChecked = e.target;
   });
+
+  // If all checkboxes are checked on page load, make sure the select-all one
+  // is checked too, otherwise keep unchecked.
+  updateSelectAll((checkboxes.length == $(checkboxes).filter(':checked').length));
 };
 
 Drupal.tableSelectRange = function (from, to, state) {
-  // We determine the looping mode based on the the order of from and to.
+  // We determine the looping mode based on the order of from and to.
   var mode = from.rowIndex > to.rowIndex ? 'previousSibling' : 'nextSibling';
 
   // Traverse through the sibling nodes.

misc/typo3/drupal-security/PharExtensionInterceptor.php

@@ -0,0 +1,79 @@
+<?php
+
+namespace Drupal\Core\Security;
+
+use TYPO3\PharStreamWrapper\Assertable;
+use TYPO3\PharStreamWrapper\Helper;
+use TYPO3\PharStreamWrapper\Exception;
+
+/**
+ * An alternate PharExtensionInterceptor to support phar-based CLI tools.
+ *
+ * @see \TYPO3\PharStreamWrapper\Interceptor\PharExtensionInterceptor
+ */
+class PharExtensionInterceptor implements Assertable {
+
+  /**
+   * Determines whether phar file is allowed to execute.
+   *
+   * The phar file is allowed to execute if:
+   * - the base file name has a ".phar" suffix.
+   * - it is the CLI tool that has invoked the interceptor.
+   *
+   * @param string $path
+   *   The path of the phar file to check.
+   * @param string $command
+   *   The command being carried out.
+   *
+   * @return bool
+   *   TRUE if the phar file is allowed to execute.
+   *
+   * @throws Exception
+   *   Thrown when the file is not allowed to execute.
+   */
+  public function assert($path, $command) {
+    if ($this->baseFileContainsPharExtension($path)) {
+      return TRUE;
+    }
+    throw new Exception(
+      sprintf(
+        'Unexpected file extension in "%s"',
+        $path
+      ),
+      1535198703
+    );
+  }
+
+  /**
+   * Determines if a path has a .phar extension or invoked execution.
+   *
+   * @param string $path
+   *   The path of the phar file to check.
+   *
+   * @return bool
+   *   TRUE if the file has a .phar extension or if the execution has been
+   *   invoked by the phar file.
+   */
+  private function baseFileContainsPharExtension($path) {
+    $baseFile = Helper::determineBaseFile($path);
+    if ($baseFile === NULL) {
+      return FALSE;
+    }
+    // If the stream wrapper is registered by invoking a phar file that does
+    // not not have .phar extension then this should be allowed. For
+    // example, some CLI tools recommend removing the extension.
+    $backtrace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS);
+    // Find the last entry in the backtrace containing a 'file' key as
+    // sometimes the last caller is executed outside the scope of a file. For
+    // example, this occurs with shutdown functions.
+    do {
+      $caller = array_pop($backtrace);
+    } while (empty($caller['file']) && !empty($backtrace));
+    if (isset($caller['file']) && $baseFile === Helper::determineBaseFile($caller['file'])) {
+      return TRUE;
+    }
+    $fileExtension = pathinfo($baseFile, PATHINFO_EXTENSION);
+    return strtolower($fileExtension) === 'phar';
+  }
+
+}

misc/typo3/phar-stream-wrapper/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 TYPO3 project - https://typo3.org/
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

misc/typo3/phar-stream-wrapper/README.md

@@ -0,0 +1,221 @@
+[![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/TYPO3/phar-stream-wrapper/badges/quality-score.png?b=v2)](https://scrutinizer-ci.com/g/TYPO3/phar-stream-wrapper/?branch=v2)
+[![Travis CI Build Status](https://travis-ci.org/TYPO3/phar-stream-wrapper.svg?branch=v2)](https://travis-ci.org/TYPO3/phar-stream-wrapper)
+[![AppVeyor Build status](https://ci.appveyor.com/api/projects/status/q4ls5tg4w1d6sf4i/branch/v2?svg=true)](https://ci.appveyor.com/project/ohader/phar-stream-wrapper)
+
+# PHP Phar Stream Wrapper
+
+## Abstract & History
+
+Based on Sam Thomas' findings concerning
+[insecure deserialization in combination with obfuscation strategies](https://blog.secarma.co.uk/labs/near-phar-dangerous-unserialization-wherever-you-are)
+allowing to hide Phar files inside valid image resources, the TYPO3 project
+decided back then to introduce a `PharStreamWrapper` to intercept invocations
+of the `phar://` stream in PHP and only allow usage for defined locations in
+the file system.
+
+Since the TYPO3 mission statement is **inspiring people to share**, we thought
+it would be helpful for others to release our `PharStreamWrapper` as standalone
+package to the PHP community.
+
+The mentioned security issue was reported to TYPO3 on 10th June 2018 by Sam Thomas
+and has been addressed concerning the specific attack vector and for this generic
+`PharStreamWrapper` in TYPO3 versions 7.6.30 LTS, 8.7.17 LTS and 9.3.1 on 12th
+July 2018.
+
+* https://blog.secarma.co.uk/labs/near-phar-dangerous-unserialization-wherever-you-are
+* https://youtu.be/GePBmsNJw6Y
+* https://typo3.org/security/advisory/typo3-psa-2018-001/
+* https://typo3.org/security/advisory/typo3-psa-2019-007/
+* https://typo3.org/security/advisory/typo3-psa-2019-008/
+
+## License
+
+In general the TYPO3 core is released under the GNU General Public License version
+2 or any later version (`GPL-2.0-or-later`). In order to avoid licensing issues and
+incompatibilities this `PharStreamWrapper` is licenced under the MIT License. In case
+you duplicate or modify source code, credits are not required but really appreciated.
+
+## Credits
+
+Thanks to [Alex Pott](https://github.com/alexpott), Drupal for creating
+back-ports of all sources in order to provide compatibility with PHP v5.3.
+
+## Installation
+
+The `PharStreamWrapper` is provided as composer package `typo3/phar-stream-wrapper`
+and has minimum requirements of PHP v5.3 ([`v2`](https://github.com/TYPO3/phar-stream-wrapper/tree/v2) branch) and PHP v7.0 ([`master`](https://github.com/TYPO3/phar-stream-wrapper) branch).
+
+### Installation for PHP v7.0
+
+```
+composer require typo3/phar-stream-wrapper ^3.0
+```
+
+### Installation for PHP v5.3
+
+```
+composer require typo3/phar-stream-wrapper ^2.0
+```
+
+## Example
+
+The following example is bundled within this package, the shown
+`PharExtensionInterceptor` denies all stream wrapper invocations files
+not having the `.phar` suffix. Interceptor logic has to be individual and
+adjusted to according requirements.
+
+```
+$behavior = new \TYPO3\PharStreamWrapper\Behavior();
+\TYPO3\PharStreamWrapper\Manager::initialize(
+    $behavior->withAssertion(new PharExtensionInterceptor())
+);
+
+if (in_array('phar', stream_get_wrappers())) {
+    stream_wrapper_unregister('phar');
+    stream_wrapper_register('phar', 'TYPO3\\PharStreamWrapper\\PharStreamWrapper');
+}
+```
+
+* `PharStreamWrapper` defined as class reference will be instantiated each time
+  `phar://` streams shall be processed.
+* `Manager` as singleton pattern being called by `PharStreamWrapper` instances
+  in order to retrieve individual behavior and settings.
+* `Behavior` holds reference to interceptor(s) that shall assert correct/allowed
+  invocation of a given `$path` for a given `$command`. Interceptors implement
+  the interface `Assertable`. Interceptors can act individually on following
+  commands or handle all of them in case not defined specifically:  
+  + `COMMAND_DIR_OPENDIR`
+  + `COMMAND_MKDIR`
+  + `COMMAND_RENAME`
+  + `COMMAND_RMDIR`
+  + `COMMAND_STEAM_METADATA`
+  + `COMMAND_STREAM_OPEN`
+  + `COMMAND_UNLINK`
+  + `COMMAND_URL_STAT`
+
+## Interceptors
+
+The following interceptor is shipped with the package and ready to use in order
+to block any Phar invocation of files not having a `.phar` suffix. Besides that
+individual interceptors are possible of course.
+
+```
+class PharExtensionInterceptor implements Assertable
+{
+    /**
+     * Determines whether the base file name has a ".phar" suffix.
+     *
+     * @param string $path
+     * @param string $command
+     * @return bool
+     * @throws Exception
+     */
+    public function assert($path, $command)
+    {
+        if ($this->baseFileContainsPharExtension($path)) {
+            return true;
+        }
+        throw new Exception(
+            sprintf(
+                'Unexpected file extension in "%s"',
+                $path
+            ),
+            1535198703
+        );
+    }
+
+    /**
+     * @param string $path
+     * @return bool
+     */
+    private function baseFileContainsPharExtension($path)
+    {
+        $baseFile = Helper::determineBaseFile($path);
+        if ($baseFile === null) {
+            return false;
+        }
+        $fileExtension = pathinfo($baseFile, PATHINFO_EXTENSION);
+        return strtolower($fileExtension) === 'phar';
+    }
+}
+```
+
+### ConjunctionInterceptor
+
+This interceptor combines multiple interceptors implementing `Assertable`.
+It succeeds when all nested interceptors succeed as well (logical `AND`).
+
+```
+$behavior = new \TYPO3\PharStreamWrapper\Behavior();
+\TYPO3\PharStreamWrapper\Manager::initialize(
+    $behavior->withAssertion(new ConjunctionInterceptor(array(
+        new PharExtensionInterceptor(),
+        new PharMetaDataInterceptor()
+    )))
+);
+```
+
+### PharExtensionInterceptor
+
+This (basic) interceptor just checks whether the invoked Phar archive has
+an according `.phar` file extension. Resolving symbolic links as well as
+Phar internal alias resolving are considered as well.
+
+```
+$behavior = new \TYPO3\PharStreamWrapper\Behavior();
+\TYPO3\PharStreamWrapper\Manager::initialize(
+    $behavior->withAssertion(new PharExtensionInterceptor())
+);
+```
+
+### PharMetaDataInterceptor
+
+This interceptor is actually checking serialized Phar meta-data against
+PHP objects and would consider a Phar archive malicious in case not only
+scalar values are found. A custom low-level `Phar\Reader` is used in order to
+avoid using PHP's `Phar` object which would trigger the initial vulnerability.
+
+```
+$behavior = new \TYPO3\PharStreamWrapper\Behavior();
+\TYPO3\PharStreamWrapper\Manager::initialize(
+    $behavior->withAssertion(new PharMetaDataInterceptor())
+);
+```
+
+## Reader
+
+* `Phar\Reader::__construct(string $fileName)`: Creates low-level reader for Phar archive
+* `Phar\Reader::resolveContainer(): Phar\Container`: Resolves model representing Phar archive
+* `Phar\Container::getStub(): Phar\Stub`: Resolves (plain PHP) stub section of Phar archive
+* `Phar\Container::getManifest(): Phar\Manifest`: Resolves parsed Phar archive manifest as
+  documented at http://php.net/manual/en/phar.fileformat.manifestfile.php
+* `Phar\Stub::getMappedAlias(): string`: Resolves internal Phar archive alias defined in stub
+  using `Phar::mapPhar('alias.phar')` - actually the plain PHP source is analyzed here
+* `Phar\Manifest::getAlias(): string` - Resolves internal Phar archive alias defined in manifest
+  using `Phar::setAlias('alias.phar')`
+* `Phar\Manifest::getMetaData(): string`: Resolves serialized Phar archive meta-data
+* `Phar\Manifest::deserializeMetaData(): mixed`: Resolves deserialized Phar archive meta-data
+  containing only scalar values - in case an object is determined, an according
+  `Phar\DeserializationException` will be thrown
+
+```
+$reader = new Phar\Reader('example.phar');
+var_dump($reader->resolveContainer()->getManifest()->deserializeMetaData());
+```
+
+## Helper
+
+* `Helper::determineBaseFile(string $path): string`: Determines base file that can be
+  accessed using the regular file system. For instance the following path
+  `phar:///home/user/bundle.phar/content.txt` would be resolved to
+  `/home/user/bundle.phar`.
+* `Helper::resetOpCache()`: Resets PHP's OPcache if enabled as work-around for
+  issues in `include()` or `require()` calls and OPcache delivering wrong
+  results. More details can be found in PHP's bug tracker, for instance like
+  https://bugs.php.net/bug.php?id=66569
+
+## Security Contact
+
+In case of finding additional security issues in the TYPO3 project or in this
+`PharStreamWrapper` package in particular, please get in touch with the
+[TYPO3 Security Team](mailto:security@typo3.org).

misc/typo3/phar-stream-wrapper/composer.json

@@ -0,0 +1,30 @@
+{
+    "name": "typo3/phar-stream-wrapper",
+    "description": "Interceptors for PHP's native phar:// stream handling",
+    "type": "library",
+    "license": "MIT",
+    "homepage": "https://typo3.org/",
+    "keywords": ["php", "phar", "stream-wrapper", "security"],
+    "require": {
+        "php": "^5.3.3|^7.0",
+        "ext-json": "*",
+        "brumann/polyfill-unserialize": "^1.0"
+    },
+    "require-dev": {
+        "ext-xdebug": "*",
+        "phpunit/phpunit": "^4.8.36"
+    },
+    "suggest": {
+        "ext-fileinfo": "For PHP builtin file type guessing, otherwise uses internal processing"
+    },
+    "autoload": {
+        "psr-4": {
+            "TYPO3\\PharStreamWrapper\\": "src/"
+        }
+    },
+    "autoload-dev": {
+        "psr-4": {
+            "TYPO3\\PharStreamWrapper\\Tests\\": "tests/"
+        }
+    }
+}

misc/typo3/phar-stream-wrapper/src/Assertable.php

@@ -0,0 +1,22 @@
+<?php
+namespace TYPO3\PharStreamWrapper;
+
+/*
+ * This file is part of the TYPO3 project.
+ *
+ * It is free software; you can redistribute it and/or modify it under the terms
+ * of the MIT License (MIT). For the full copyright and license information,
+ * please read the LICENSE file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+interface Assertable
+{
+    /**
+     * @param string $path
+     * @param string $command
+     * @return bool
+     */
+    public function assert($path, $command);
+}

misc/typo3/phar-stream-wrapper/src/Behavior.php

@@ -0,0 +1,124 @@
+<?php
+namespace TYPO3\PharStreamWrapper;
+
+/*
+ * This file is part of the TYPO3 project.
+ *
+ * It is free software; you can redistribute it and/or modify it under the terms
+ * of the MIT License (MIT). For the full copyright and license information,
+ * please read the LICENSE file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+class Behavior implements Assertable
+{
+    const COMMAND_DIR_OPENDIR = 'dir_opendir';
+    const COMMAND_MKDIR = 'mkdir';
+    const COMMAND_RENAME = 'rename';
+    const COMMAND_RMDIR = 'rmdir';
+    const COMMAND_STEAM_METADATA = 'stream_metadata';
+    const COMMAND_STREAM_OPEN = 'stream_open';
+    const COMMAND_UNLINK = 'unlink';
+    const COMMAND_URL_STAT = 'url_stat';
+
+    /**
+     * @var string[]
+     */
+    private $availableCommands = array(
+        self::COMMAND_DIR_OPENDIR,
+        self::COMMAND_MKDIR,
+        self::COMMAND_RENAME,
+        self::COMMAND_RMDIR,
+        self::COMMAND_STEAM_METADATA,
+        self::COMMAND_STREAM_OPEN,
+        self::COMMAND_UNLINK,
+        self::COMMAND_URL_STAT,
+    );
+
+    /**
+     * @var Assertable[]
+     */
+    private $assertions;
+
+    /**
+     * @param Assertable $assertable
+     * @return static
+     */
+    public function withAssertion(Assertable $assertable)
+    {
+        $commands = func_get_args();
+        array_shift($commands);
+        $this->assertCommands($commands);
+        $commands = $commands ?: $this->availableCommands;
+
+        $target = clone $this;
+        foreach ($commands as $command) {
+            $target->assertions[$command] = $assertable;
+        }
+        return $target;
+    }
+
+    /**
+     * @param string $path
+     * @param string $command
+     * @return bool
+     */
+    public function assert($path, $command)
+    {
+        $this->assertCommand($command);
+        $this->assertAssertionCompleteness();
+
+        return $this->assertions[$command]->assert($path, $command);
+    }
+
+    /**
+     * @param array $commands
+     */
+    private function assertCommands(array $commands)
+    {
+        $unknownCommands = array_diff($commands, $this->availableCommands);
+        if (empty($unknownCommands)) {
+            return;
+        }
+        throw new \LogicException(
+            sprintf(
+                'Unknown commands: %s',
+                implode(', ', $unknownCommands)
+            ),
+            1535189881
+        );
+    }
+
+    private function assertCommand($command)
+    {
+        if (in_array($command, $this->availableCommands, true)) {
+            return;
+        }
+        throw new \LogicException(
+            sprintf(
+                'Unknown command "%s"',
+                $command
+            ),
+            1535189882
+        );
+    }
+
+    private function assertAssertionCompleteness()
+    {
+        $undefinedAssertions = array_diff(
+            $this->availableCommands,
+            array_keys($this->assertions)
+        );
+        if (empty($undefinedAssertions)) {
+            return;
+        }
+        throw new \LogicException(
+            sprintf(
+                'Missing assertions for commands: %s',
+                implode(', ', $undefinedAssertions)
+            ),
+            1535189883
+        );
+    }
+}

misc/typo3/phar-stream-wrapper/src/Collectable.php

@@ -0,0 +1,37 @@
+<?php
+namespace TYPO3\PharStreamWrapper;
+
+/*
+ * This file is part of the TYPO3 project.
+ *
+ * It is free software; you can redistribute it and/or modify it under the terms
+ * of the MIT License (MIT). For the full copyright and license information,
+ * please read the LICENSE file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use TYPO3\PharStreamWrapper\Resolver\PharInvocation;
+
+interface Collectable
+{
+    /**
+     * @param PharInvocation $invocation
+     * @return bool
+     */
+    public function has(PharInvocation $invocation);
+
+    /**
+     * @param PharInvocation $invocation
+     * @param null $flags
+     * @return bool
+     */
+    public function collect(PharInvocation $invocation, $flags = null);
+
+    /**
+     * @param callable $callback
+     * @param bool $reverse
+     * @return null|PharInvocation
+     */
+    public function findByCallback($callback, $reverse = false);
+}

misc/typo3/phar-stream-wrapper/src/Exception.php

@@ -0,0 +1,16 @@
+<?php
+namespace TYPO3\PharStreamWrapper;
+
+/*
+ * This file is part of the TYPO3 project.
+ *
+ * It is free software; you can redistribute it and/or modify it under the terms
+ * of the MIT License (MIT). For the full copyright and license information,
+ * please read the LICENSE file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+class Exception extends \RuntimeException
+{
+}

misc/typo3/phar-stream-wrapper/src/Helper.php

@@ -0,0 +1,199 @@
+<?php
+namespace TYPO3\PharStreamWrapper;
+
+/*
+ * This file is part of the TYPO3 project.
+ *
+ * It is free software; you can redistribute it and/or modify it under the terms
+ * of the MIT License (MIT). For the full copyright and license information,
+ * please read the LICENSE file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+/**
+ * Helper provides low-level tools on file name resolving. However it does not
+ * (and should not) maintain any runtime state information. In order to resolve
+ * Phar archive paths according resolvers have to be used.
+ *
+ * @see \TYPO3\PharStreamWrapper\Resolvable::resolve()
+ */
+class Helper
+{
+    /*
+     * Resets PHP's OPcache if enabled as work-around for issues in `include()`
+     * or `require()` calls and OPcache delivering wrong results.
+     *
+     * @see https://bugs.php.net/bug.php?id=66569
+     */
+    public static function resetOpCache()
+    {
+        if (function_exists('opcache_reset')
+            && function_exists('opcache_get_status')
+        ) {
+            $status = opcache_get_status();
+            if (!empty($status['opcache_enabled'])) {
+                opcache_reset();
+            }
+        }
+    }
+
+    /**
+     * Determines base file that can be accessed using the regular file system.
+     * For e.g. "phar:///home/user/bundle.phar/content.txt" that would result
+     * into "/home/user/bundle.phar".
+     *
+     * @param string $path
+     * @return string|null
+     */
+    public static function determineBaseFile($path)
+    {
+        $parts = explode('/', static::normalizePath($path));
+
+        while (count($parts)) {
+            $currentPath = implode('/', $parts);
+            if (@is_file($currentPath) && realpath($currentPath) !== false) {
+                return $currentPath;
+            }
+            array_pop($parts);
+        }
+
+        return null;
+    }
+
+    /**
+     * @param string $path
+     * @return bool
+     */
+    public static function hasPharPrefix($path)
+    {
+        return stripos($path, 'phar://') === 0;
+    }
+
+    /**
+     * @param string $path
+     * @return string
+     */
+    public static function removePharPrefix($path)
+    {
+        $path = trim($path);
+        if (!static::hasPharPrefix($path)) {
+            return $path;
+        }
+        return substr($path, 7);
+    }
+
+    /**
+     * Normalizes a path, removes phar:// prefix, fixes Windows directory
+     * separators. Result is without trailing slash.
+     *
+     * @param string $path
+     * @return string
+     */
+    public static function normalizePath($path)
+    {
+        return rtrim(
+            static::normalizeWindowsPath(
+                static::removePharPrefix($path)
+            ),
+            '/'
+        );
+    }
+
+    /**
+     * Fixes a path for windows-backslashes and reduces double-slashes to single slashes
+     *
+     * @param string $path File path to process
+     * @return string
+     */
+    public static function normalizeWindowsPath($path)
+    {
+        return str_replace('\\', '/', $path);
+    }
+
+    /**
+     * Resolves all dots, slashes and removes spaces after or before a path...
+     *
+     * @param string $path Input string
+     * @return string Canonical path, always without trailing slash
+     */
+    private static function getCanonicalPath($path)
+    {
+        $path = static::normalizeWindowsPath($path);
+
+        $absolutePathPrefix = '';
+        if (static::isAbsolutePath($path)) {
+            if (static::isWindows() && strpos($path, ':/') === 1) {
+                $absolutePathPrefix = substr($path, 0, 3);
+                $path = substr($path, 3);
+            } else {
+                $path = ltrim($path, '/');
+                $absolutePathPrefix = '/';
+            }
+        }
+
+        $pathParts = explode('/', $path);
+        $pathPartsLength = count($pathParts);
+        for ($partCount = 0; $partCount < $pathPartsLength; $partCount++) {
+            // double-slashes in path: remove element
+            if ($pathParts[$partCount] === '') {
+                array_splice($pathParts, $partCount, 1);
+                $partCount--;
+                $pathPartsLength--;
+            }
+            // "." in path: remove element
+            if ((isset($pathParts[$partCount]) ? $pathParts[$partCount] : '') === '.') {
+                array_splice($pathParts, $partCount, 1);
+                $partCount--;
+                $pathPartsLength--;
+            }
+            // ".." in path:
+            if ((isset($pathParts[$partCount]) ? $pathParts[$partCount] : '') === '..') {
+                if ($partCount === 0) {
+                    array_splice($pathParts, $partCount, 1);
+                    $partCount--;
+                    $pathPartsLength--;
+                } elseif ($partCount >= 1) {
+                    // Rremove this and previous element
+                    array_splice($pathParts, $partCount - 1, 2);
+                    $partCount -= 2;
+                    $pathPartsLength -= 2;
+                } elseif ($absolutePathPrefix) {
+                    // can't go higher than root dir
+                    // simply remove this part and continue
+                    array_splice($pathParts, $partCount, 1);
+                    $partCount--;
+                    $pathPartsLength--;
+                }
+            }
+        }
+
+        return $absolutePathPrefix . implode('/', $pathParts);
+    }
+
+    /**
+     * Checks if the $path is absolute or relative (detecting either '/' or
+     * 'x:/' as first part of string) and returns TRUE if so.
+     *
+     * @param string $path File path to evaluate
+     * @return bool
+     */
+    private static function isAbsolutePath($path)
+    {
+        // Path starting with a / is always absolute, on every system
+        // On Windows also a path starting with a drive letter is absolute: X:/
+        return (isset($path[0]) ? $path[0] : null) === '/'
+            || static::isWindows() && (
+                strpos($path, ':/') === 1
+                || strpos($path, ':\\') === 1
+            );
+    }
+
+    /**
+     * @return bool
+     */
+    private static function isWindows()
+    {
+        return stripos(PHP_OS, 'WIN') === 0;
+    }
+}

misc/typo3/phar-stream-wrapper/src/Interceptor/ConjunctionInterceptor.php

@@ -0,0 +1,88 @@
+<?php
+namespace TYPO3\PharStreamWrapper\Interceptor;
+
+/*
+ * This file is part of the TYPO3 project.
+ *
+ * It is free software; you can redistribute it and/or modify it under the terms
+ * of the MIT License (MIT). For the full copyright and license information,
+ * please read the LICENSE file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use TYPO3\PharStreamWrapper\Assertable;
+use TYPO3\PharStreamWrapper\Exception;
+
+class ConjunctionInterceptor implements Assertable
+{
+    /**
+     * @var Assertable[]
+     */
+    private $assertions;
+
+    public function __construct(array $assertions)
+    {
+        $this->assertAssertions($assertions);
+        $this->assertions = $assertions;
+    }
+
+    /**
+     * Executes assertions based on all contained assertions.
+     *
+     * @param string $path
+     * @param string $command
+     * @return bool
+     * @throws Exception
+     */
+    public function assert($path, $command)
+    {
+        if ($this->invokeAssertions($path, $command)) {
+            return true;
+        }
+        throw new Exception(
+            sprintf(
+                'Assertion failed in "%s"',
+                $path
+            ),
+            1539625084
+        );
+    }
+
+    /**
+     * @param Assertable[] $assertions
+     */
+    private function assertAssertions(array $assertions)
+    {
+        foreach ($assertions as $assertion) {
+            if (!$assertion instanceof Assertable) {
+                throw new \InvalidArgumentException(
+                    sprintf(
+                        'Instance %s must implement Assertable',
+                        get_class($assertion)
+                    ),
+                    1539624719
+                );
+            }
+        }
+    }
+
+    /**
+     * @param string $path
+     * @param string $command
+     * @return bool
+     */
+    private function invokeAssertions($path, $command)
+    {
+        try {
+            foreach ($this->assertions as $assertion) {
+                if (!$assertion->assert($path, $command)) {
+                    return false;
+                }
+            }
+        } catch (Exception $exception) {
+            return false;
+        }
+        return true;
+    }
+}

misc/typo3/phar-stream-wrapper/src/Interceptor/PharExtensionInterceptor.php

@@ -0,0 +1,55 @@
+<?php
+namespace TYPO3\PharStreamWrapper\Interceptor;
+
+/*
+ * This file is part of the TYPO3 project.
+ *
+ * It is free software; you can redistribute it and/or modify it under the terms
+ * of the MIT License (MIT). For the full copyright and license information,
+ * please read the LICENSE file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use TYPO3\PharStreamWrapper\Assertable;
+use TYPO3\PharStreamWrapper\Exception;
+use TYPO3\PharStreamWrapper\Manager;
+
+class PharExtensionInterceptor implements Assertable
+{
+    /**
+     * Determines whether the base file name has a ".phar" suffix.
+     *
+     * @param string $path
+     * @param string $command
+     * @return bool
+     * @throws Exception
+     */
+    public function assert($path, $command)
+    {
+        if ($this->baseFileContainsPharExtension($path)) {
+            return true;
+        }
+        throw new Exception(
+            sprintf(
+                'Unexpected file extension in "%s"',
+                $path
+            ),
+            1535198703
+        );
+    }
+
+    /**
+     * @param string $path
+     * @return bool
+     */
+    private function baseFileContainsPharExtension($path)
+    {
+        $invocation = Manager::instance()->resolve($path);
+        if ($invocation === null) {
+            return false;
+        }
+        $fileExtension = pathinfo($invocation->getBaseName(), PATHINFO_EXTENSION);
+        return strtolower($fileExtension) === 'phar';
+    }
+}

misc/typo3/phar-stream-wrapper/src/Interceptor/PharMetaDataInterceptor.php

@@ -0,0 +1,73 @@
+<?php
+namespace TYPO3\PharStreamWrapper\Interceptor;
+
+/*
+ * This file is part of the TYPO3 project.
+ *
+ * It is free software; you can redistribute it and/or modify it under the terms
+ * of the MIT License (MIT). For the full copyright and license information,
+ * please read the LICENSE file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use TYPO3\PharStreamWrapper\Assertable;
+use TYPO3\PharStreamWrapper\Exception;
+use TYPO3\PharStreamWrapper\Manager;
+use TYPO3\PharStreamWrapper\Phar\DeserializationException;
+use TYPO3\PharStreamWrapper\Phar\Reader;
+
+/**
+ * @internal Experimental implementation of checking against serialized objects in Phar meta-data
+ * @internal This functionality has not been 100% pentested...
+ */
+class PharMetaDataInterceptor implements Assertable
+{
+    /**
+     * Determines whether the according Phar archive contains
+     * (potential insecure) serialized objects.
+     *
+     * @param string $path
+     * @param string $command
+     * @return bool
+     * @throws Exception
+     */
+    public function assert($path, $command)
+    {
+        if ($this->baseFileDoesNotHaveMetaDataIssues($path)) {
+            return true;
+        }
+        throw new Exception(
+            sprintf(
+                'Problematic meta-data in "%s"',
+                $path
+            ),
+            1539632368
+        );
+    }
+
+    /**
+     * @param string $path
+     * @return bool
+     */
+    private function baseFileDoesNotHaveMetaDataIssues($path)
+    {
+        $invocation = Manager::instance()->resolve($path);
+        if ($invocation === null) {
+            return false;
+        }
+        // directly return in case invocation was checked before
+        if ($invocation->getVariable(__CLASS__) === true) {
+            return true;
+        }
+        // otherwise analyze meta-data
+        try {
+            $reader = new Reader($invocation->getBaseName());
+            $reader->resolveContainer()->getManifest()->deserializeMetaData();
+            $invocation->setVariable(__CLASS__, true);
+        } catch (DeserializationException $exception) {
+            return false;
+        }
+        return true;
+    }
+}

misc/typo3/phar-stream-wrapper/src/Manager.php

@@ -0,0 +1,135 @@
+<?php
+namespace TYPO3\PharStreamWrapper;
+
+/*
+ * This file is part of the TYPO3 project.
+ *
+ * It is free software; you can redistribute it and/or modify it under the terms
+ * of the MIT License (MIT). For the full copyright and license information,
+ * please read the LICENSE file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use TYPO3\PharStreamWrapper\Resolver\PharInvocation;
+use TYPO3\PharStreamWrapper\Resolver\PharInvocationCollection;
+use TYPO3\PharStreamWrapper\Resolver\PharInvocationResolver;
+
+class Manager
+{
+    /**
+     * @var self
+     */
+    private static $instance;
+
+    /**
+     * @var Behavior
+     */
+    private $behavior;
+
+    /**
+     * @var Resolvable
+     */
+    private $resolver;
+
+    /**
+     * @var Collectable
+     */
+    private $collection;
+
+    /**
+     * @param Behavior $behaviour
+     * @param Resolvable $resolver
+     * @param Collectable $collection
+     * @return self
+     */
+    public static function initialize(
+        Behavior $behaviour,
+        Resolvable $resolver = null,
+        Collectable $collection = null
+    ) {
+        if (self::$instance === null) {
+            self::$instance = new self($behaviour, $resolver, $collection);
+            return self::$instance;
+        }
+        throw new \LogicException(
+            'Manager can only be initialized once',
+            1535189871
+        );
+    }
+
+    /**
+     * @return self
+     */
+    public static function instance()
+    {
+        if (self::$instance !== null) {
+            return self::$instance;
+        }
+        throw new \LogicException(
+            'Manager needs to be initialized first',
+            1535189872
+        );
+    }
+
+    /**
+     * @return bool
+     */
+    public static function destroy()
+    {
+        if (self::$instance === null) {
+            return false;
+        }
+        self::$instance = null;
+        return true;
+    }
+
+    /**
+     * @param Behavior $behaviour
+     * @param Resolvable $resolver
+     * @param Collectable $collection
+     */
+    private function __construct(
+        Behavior $behaviour,
+        Resolvable $resolver = null,
+        Collectable $collection = null
+    ) {
+        if ($collection === null) {
+            $collection = new PharInvocationCollection();
+        }
+        if ($resolver === null) {
+            $resolver = new PharInvocationResolver();
+        }
+        $this->collection = $collection;
+        $this->resolver = $resolver;
+        $this->behavior = $behaviour;
+    }
+
+    /**
+     * @param string $path
+     * @param string $command
+     * @return bool
+     */
+    public function assert($path, $command)
+    {
+        return $this->behavior->assert($path, $command);
+    }
+
+    /**
+     * @param string $path
+     * @param null|int $flags
+     * @return null|PharInvocation
+     */
+    public function resolve($path, $flags = null)
+    {
+        return $this->resolver->resolve($path, $flags);
+    }
+
+    /**
+     * @return Collectable
+     */
+    public function getCollection()
+    {
+        return $this->collection;
+    }
+}

misc/typo3/phar-stream-wrapper/src/Phar/Container.php

@@ -0,0 +1,59 @@
+<?php
+namespace TYPO3\PharStreamWrapper\Phar;
+
+/*
+ * This file is part of the TYPO3 project.
+ *
+ * It is free software; you can redistribute it and/or modify it under the terms
+ * of the MIT License (MIT). For the full copyright and license information,
+ * please read the LICENSE file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+class Container
+{
+    /**
+     * @var Stub
+     */
+    private $stub;
+
+    /**
+     * @var Manifest
+     */
+    private $manifest;
+
+    /**
+     * @param Stub $stub
+     * @param Manifest $manifest
+     */
+    public function __construct(Stub $stub, Manifest $manifest)
+    {
+        $this->stub = $stub;
+        $this->manifest = $manifest;
+    }
+
+    /**
+     * @return Stub
+     */
+    public function getStub()
+    {
+        return $this->stub;
+    }
+
+    /**
+     * @return Manifest
+     */
+    public function getManifest()
+    {
+        return $this->manifest;
+    }
+
+    /**
+     * @return string
+     */
+    public function getAlias()
+    {
+        return $this->manifest->getAlias() ?: $this->stub->getMappedAlias();
+    }
+}

misc/typo3/phar-stream-wrapper/src/Phar/DeserializationException.php

@@ -0,0 +1,18 @@
+<?php
+namespace TYPO3\PharStreamWrapper\Phar;
+
+/*
+ * This file is part of the TYPO3 project.
+ *
+ * It is free software; you can redistribute it and/or modify it under the terms
+ * of the MIT License (MIT). For the full copyright and license information,
+ * please read the LICENSE file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use TYPO3\PharStreamWrapper\Exception;
+
+class DeserializationException extends Exception
+{
+}

misc/typo3/phar-stream-wrapper/src/Phar/Manifest.php

@@ -0,0 +1,176 @@
+<?php
+namespace TYPO3\PharStreamWrapper\Phar;
+
+/*
+ * This file is part of the TYPO3 project.
+ *
+ * It is free software; you can redistribute it and/or modify it under the terms
+ * of the MIT License (MIT). For the full copyright and license information,
+ * please read the LICENSE file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use Brumann\Polyfill\Unserialize;
+
+class Manifest
+{
+    /**
+     * @param string $content
+     * @return self
+     * @see http://php.net/manual/en/phar.fileformat.phar.php
+     */
+    public static function fromContent($content)
+    {
+        $target = new static();
+        $target->manifestLength = Reader::resolveFourByteLittleEndian($content, 0);
+        $target->amountOfFiles = Reader::resolveFourByteLittleEndian($content, 4);
+        $target->flags = Reader::resolveFourByteLittleEndian($content, 10);
+        $target->aliasLength = Reader::resolveFourByteLittleEndian($content, 14);
+        $target->alias = substr($content, 18, $target->aliasLength);
+        $target->metaDataLength = Reader::resolveFourByteLittleEndian($content, 18 + $target->aliasLength);
+        $target->metaData = substr($content, 22 + $target->aliasLength, $target->metaDataLength);
+
+        $apiVersionNibbles = Reader::resolveTwoByteBigEndian($content, 8);
+        $target->apiVersion = implode('.', array(
+            ($apiVersionNibbles & 0xf000) >> 12,
+            ($apiVersionNibbles & 0x0f00) >> 8,
+            ($apiVersionNibbles & 0x00f0) >> 4,
+        ));
+
+        return $target;
+    }
+
+    /**
+     * @var int
+     */
+    private $manifestLength;
+
+    /**
+     * @var int
+     */
+    private $amountOfFiles;
+
+    /**
+     * @var string
+     */
+    private $apiVersion;
+
+    /**
+     * @var int
+     */
+    private $flags;
+
+    /**
+     * @var int
+     */
+    private $aliasLength;
+
+    /**
+     * @var string
+     */
+    private $alias;
+
+    /**
+     * @var int
+     */
+    private $metaDataLength;
+
+    /**
+     * @var string
+     */
+    private $metaData;
+
+    /**
+     * Avoid direct instantiation.
+     */
+    private function __construct()
+    {
+    }
+
+    /**
+     * @return int
+     */
+    public function getManifestLength()
+    {
+        return $this->manifestLength;
+    }
+
+    /**
+     * @return int
+     */
+    public function getAmountOfFiles()
+    {
+        return $this->amountOfFiles;
+    }
+
+    /**
+     * @return string
+     */
+    public function getApiVersion()
+    {
+        return $this->apiVersion;
+    }
+
+    /**
+     * @return int
+     */
+    public function getFlags()
+    {
+        return $this->flags;
+    }
+
+    /**
+     * @return int
+     */
+    public function getAliasLength()
+    {
+        return $this->aliasLength;
+    }
+
+    /**
+     * @return string
+     */
+    public function getAlias()
+    {
+        return $this->alias;
+    }
+
+    /**
+     * @return int
+     */
+    public function getMetaDataLength()
+    {
+        return $this->metaDataLength;
+    }
+
+    /**
+     * @return string
+     */
+    public function getMetaData()
+    {
+        return $this->metaData;
+    }
+
+    /**
+     * @return mixed|null
+     */
+    public function deserializeMetaData()
+    {
+        if (empty($this->metaData)) {
+            return null;
+        }
+
+        $result = Unserialize::unserialize($this->metaData, array('allowed_classes' => false));
+
+        $serialized = json_encode($result);
+        if (strpos($serialized, '__PHP_Incomplete_Class_Name') !== false) {
+            throw new DeserializationException(
+                'Meta-data contains serialized object',
+                1539623382
+            );
+        }
+
+        return $result;
+    }
+}

misc/typo3/phar-stream-wrapper/src/Phar/Reader.php

@@ -0,0 +1,254 @@
+<?php
+namespace TYPO3\PharStreamWrapper\Phar;
+
+/*
+ * This file is part of the TYPO3 project.
+ *
+ * It is free software; you can redistribute it and/or modify it under the terms
+ * of the MIT License (MIT). For the full copyright and license information,
+ * please read the LICENSE file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+class Reader
+{
+    /**
+     * @var string
+     */
+    private $fileName;
+
+    /**
+     * Mime-type in order to use zlib, bzip2 or no compression.
+     * In case ext-fileinfo is not present only the relevant types
+     * 'application/x-gzip' and 'application/x-bzip2' are assigned
+     * to this class property.
+     *
+     * @var string
+     */
+    private $fileType;
+
+    /**
+     * @param string $fileName
+     */
+    public function __construct($fileName)
+    {
+        if (strpos($fileName, '://') !== false) {
+            throw new ReaderException(
+                'File name must not contain stream prefix',
+                1539623708
+            );
+        }
+
+        $this->fileName = $fileName;
+        $this->fileType = $this->determineFileType();
+    }
+
+    /**
+     * @return Container
+     */
+    public function resolveContainer()
+    {
+        $data = $this->extractData($this->resolveStream() . $this->fileName);
+
+        if ($data['stubContent'] === null) {
+            throw new ReaderException(
+                'Cannot resolve stub',
+                1547807881
+            );
+        }
+        if ($data['manifestContent'] === null || $data['manifestLength'] === null) {
+            throw new ReaderException(
+                'Cannot resolve manifest',
+                1547807882
+            );
+        }
+        if (strlen($data['manifestContent']) < $data['manifestLength']) {
+            throw new ReaderException(
+                sprintf(
+                    'Exected manifest length %d, got %d',
+                    strlen($data['manifestContent']),
+                    $data['manifestLength']
+                ),
+                1547807883
+            );
+        }
+
+        return new Container(
+            Stub::fromContent($data['stubContent']),
+            Manifest::fromContent($data['manifestContent'])
+        );
+    }
+
+    /**
+     * @param string $fileName e.g. '/path/file.phar' or 'compress.zlib:///path/file.phar'
+     * @return array
+     */
+    private function extractData($fileName)
+    {
+        $stubContent = null;
+        $manifestContent = null;
+        $manifestLength = null;
+
+        $resource = fopen($fileName, 'r');
+        if (!is_resource($resource)) {
+            throw new ReaderException(
+                sprintf('Resource %s could not be opened', $fileName),
+                1547902055
+            );
+        }
+
+        while (!feof($resource)) {
+            $line = fgets($resource);
+            // stop reading file when manifest can be extracted
+            if ($manifestLength !== null && $manifestContent !== null && strlen($manifestContent) >= $manifestLength) {
+                break;
+            }
+
+            $manifestPosition = strpos($line, '__HALT_COMPILER();');
+
+            // first line contains start of manifest
+            if ($stubContent === null && $manifestContent === null && $manifestPosition !== false) {
+                $stubContent = substr($line, 0, $manifestPosition - 1);
+                $manifestContent = preg_replace('#^.*__HALT_COMPILER\(\);(?>[ \n]\?>(?>\r\n|\n)?)?#', '', $line);
+                $manifestLength = $this->resolveManifestLength($manifestContent);
+            // line contains start of stub
+            } elseif ($stubContent === null) {
+                $stubContent = $line;
+            // line contains start of manifest
+            } elseif ($manifestContent === null && $manifestPosition !== false) {
+                $manifestContent = preg_replace('#^.*__HALT_COMPILER\(\);(?>[ \n]\?>(?>\r\n|\n)?)?#', '', $line);
+                $manifestLength = $this->resolveManifestLength($manifestContent);
+            // manifest has been started (thus is cannot be stub anymore), add content
+            } elseif ($manifestContent !== null) {
+                $manifestContent .= $line;
+                $manifestLength = $this->resolveManifestLength($manifestContent);
+            // stub has been started (thus cannot be manifest here, yet), add content
+            } elseif ($stubContent !== null) {
+                $stubContent .= $line;
+            }
+        }
+        fclose($resource);
+
+        return array(
+            'stubContent' => $stubContent,
+            'manifestContent' => $manifestContent,
+            'manifestLength' => $manifestLength,
+        );
+    }
+
+    /**
+     * Resolves stream in order to handle compressed Phar archives.
+     *
+     * @return string
+     */
+    private function resolveStream()
+    {
+        if ($this->fileType === 'application/x-gzip' || $this->fileType === 'application/gzip') {
+            return 'compress.zlib://';
+        } elseif ($this->fileType === 'application/x-bzip2') {
+            return 'compress.bzip2://';
+        }
+        return '';
+    }
+
+    /**
+     * @return string
+     */
+    private function determineFileType()
+    {
+        if (class_exists('\\finfo')) {
+            $fileInfo = new \finfo();
+            return $fileInfo->file($this->fileName, FILEINFO_MIME_TYPE);
+        }
+        return $this->determineFileTypeByHeader();
+    }
+
+    /**
+     * In case ext-fileinfo is not present only the relevant types
+     * 'application/x-gzip' and 'application/x-bzip2' are resolved.
+     *
+     * @return string
+     */
+    private function determineFileTypeByHeader()
+    {
+        $resource = fopen($this->fileName, 'r');
+        if (!is_resource($resource)) {
+            throw new ReaderException(
+                sprintf('Resource %s could not be opened', $this->fileName),
+                1557753055
+            );
+        }
+        $header = fgets($resource, 4);
+        fclose($resource);
+        $mimeType = '';
+        if (strpos($header, "\x42\x5a\x68") === 0) {
+            $mimeType = 'application/x-bzip2';
+        } elseif (strpos($header, "\x1f\x8b") === 0) {
+            $mimeType = 'application/x-gzip';
+        }
+        return $mimeType;
+    }
+
+    /**
+     * @param string $content
+     * @return int|null
+     */
+    private function resolveManifestLength($content)
+    {
+        if (strlen($content) < 4) {
+            return null;
+        }
+        return static::resolveFourByteLittleEndian($content, 0);
+    }
+
+    /**
+     * @param string $content
+     * @param int $start
+     * @return int
+     */
+    public static function resolveFourByteLittleEndian($content, $start)
+    {
+        $payload = substr($content, $start, 4);
+        if (!is_string($payload)) {
+            throw new ReaderException(
+                sprintf('Cannot resolve value at offset %d', $start),
+                1539614260
+            );
+        }
+
+        $value = unpack('V', $payload);
+        if (!isset($value[1])) {
+            throw new ReaderException(
+                sprintf('Cannot resolve value at offset %d', $start),
+                1539614261
+            );
+        }
+        return $value[1];
+    }
+
+    /**
+     * @param string $content
+     * @param int $start
+     * @return int
+     */
+    public static function resolveTwoByteBigEndian($content, $start)
+    {
+        $payload = substr($content, $start, 2);
+        if (!is_string($payload)) {
+            throw new ReaderException(
+                sprintf('Cannot resolve value at offset %d', $start),
+                1539614263
+            );
+        }
+
+        $value = unpack('n', $payload);
+        if (!isset($value[1])) {
+            throw new ReaderException(
+                sprintf('Cannot resolve value at offset %d', $start),
+                1539614264
+            );
+        }
+        return $value[1];
+    }
+}

misc/typo3/phar-stream-wrapper/src/Phar/ReaderException.php

@@ -0,0 +1,18 @@
+<?php
+namespace TYPO3\PharStreamWrapper\Phar;
+
+/*
+ * This file is part of the TYPO3 project.
+ *
+ * It is free software; you can redistribute it and/or modify it under the terms
+ * of the MIT License (MIT). For the full copyright and license information,
+ * please read the LICENSE file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use TYPO3\PharStreamWrapper\Exception;
+
+class ReaderException extends Exception
+{
+}

misc/typo3/phar-stream-wrapper/src/Phar/Stub.php

@@ -0,0 +1,65 @@
+<?php
+namespace TYPO3\PharStreamWrapper\Phar;
+
+/*
+ * This file is part of the TYPO3 project.
+ *
+ * It is free software; you can redistribute it and/or modify it under the terms
+ * of the MIT License (MIT). For the full copyright and license information,
+ * please read the LICENSE file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+/**
+ * @internal Experimental implementation of Phar archive internals
+ */
+class Stub
+{
+    /**
+     * @param string $content
+     * @return self
+     */
+    public static function fromContent($content)
+    {
+        $target = new static();
+        $target->content = $content;
+
+        if (
+            stripos($content, 'Phar::mapPhar(') !== false
+            && preg_match('#Phar\:\:mapPhar\(([^)]+)\)#', $content, $matches)
+        ) {
+            // remove spaces, single & double quotes
+            // @todo `'my' . 'alias' . '.phar'` is not evaluated here
+            $target->mappedAlias = trim($matches[1], ' \'"');
+        }
+
+        return $target;
+    }
+
+    /**
+     * @var string
+     */
+    private $content;
+
+    /**
+     * @var string
+     */
+    private $mappedAlias = '';
+
+    /**
+     * @return string
+     */
+    public function getContent()
+    {
+        return $this->content;
+    }
+
+    /**
+     * @return string
+     */
+    public function getMappedAlias()
+    {
+        return $this->mappedAlias;
+    }
+}

misc/typo3/phar-stream-wrapper/src/PharStreamWrapper.php

@@ -0,0 +1,511 @@
+<?php
+namespace TYPO3\PharStreamWrapper;
+
+/*
+ * This file is part of the TYPO3 project.
+ *
+ * It is free software; you can redistribute it and/or modify it under the terms
+ * of the MIT License (MIT). For the full copyright and license information,
+ * please read the LICENSE file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use TYPO3\PharStreamWrapper\Resolver\PharInvocation;
+
+class PharStreamWrapper
+{
+    /**
+     * Internal stream constants that are not exposed to PHP, but used...
+     * @see https://github.com/php/php-src/blob/e17fc0d73c611ad0207cac8a4a01ded38251a7dc/main/php_streams.h
+     */
+    const STREAM_OPEN_FOR_INCLUDE = 128;
+
+    /**
+     * @var resource
+     */
+    public $context;
+
+    /**
+     * @var resource
+     */
+    protected $internalResource;
+
+    /**
+     * @var PharInvocation
+     */
+    protected $invocation;
+
+    /**
+     * @return bool
+     */
+    public function dir_closedir()
+    {
+        if (!is_resource($this->internalResource)) {
+            return false;
+        }
+
+        $this->invokeInternalStreamWrapper(
+            'closedir',
+            $this->internalResource
+        );
+        return !is_resource($this->internalResource);
+    }
+
+    /**
+     * @param string $path
+     * @param int $options
+     * @return bool
+     */
+    public function dir_opendir($path, $options)
+    {
+        $this->assert($path, Behavior::COMMAND_DIR_OPENDIR);
+        $this->internalResource = $this->invokeInternalStreamWrapper(
+            'opendir',
+            $path,
+            $this->context
+        );
+        return is_resource($this->internalResource);
+    }
+
+    /**
+     * @return string|false
+     */
+    public function dir_readdir()
+    {
+        return $this->invokeInternalStreamWrapper(
+            'readdir',
+            $this->internalResource
+        );
+    }
+
+    /**
+     * @return bool
+     */
+    public function dir_rewinddir()
+    {
+        if (!is_resource($this->internalResource)) {
+            return false;
+        }
+
+        $this->invokeInternalStreamWrapper(
+            'rewinddir',
+            $this->internalResource
+        );
+        return is_resource($this->internalResource);
+    }
+
+    /**
+     * @param string $path
+     * @param int $mode
+     * @param int $options
+     * @return bool
+     */
+    public function mkdir($path, $mode, $options)
+    {
+        $this->assert($path, Behavior::COMMAND_MKDIR);
+        return $this->invokeInternalStreamWrapper(
+            'mkdir',
+            $path,
+            $mode,
+            (bool) ($options & STREAM_MKDIR_RECURSIVE),
+            $this->context
+        );
+    }
+
+    /**
+     * @param string $path_from
+     * @param string $path_to
+     * @return bool
+     */
+    public function rename($path_from, $path_to)
+    {
+        $this->assert($path_from, Behavior::COMMAND_RENAME);
+        $this->assert($path_to, Behavior::COMMAND_RENAME);
+        return $this->invokeInternalStreamWrapper(
+            'rename',
+            $path_from,
+            $path_to,
+            $this->context
+        );
+    }
+
+    /**
+     * @param string $path
+     * @param int $options
+     * @return bool
+     */
+    public function rmdir($path, $options)
+    {
+        $this->assert($path, Behavior::COMMAND_RMDIR);
+        return $this->invokeInternalStreamWrapper(
+            'rmdir',
+            $path,
+            $this->context
+        );
+    }
+
+    /**
+     * @param int $cast_as
+     */
+    public function stream_cast($cast_as)
+    {
+        throw new Exception(
+            'Method stream_select() cannot be used',
+            1530103999
+        );
+    }
+
+    public function stream_close()
+    {
+        $this->invokeInternalStreamWrapper(
+            'fclose',
+            $this->internalResource
+        );
+    }
+
+    /**
+     * @return bool
+     */
+    public function stream_eof()
+    {
+        return $this->invokeInternalStreamWrapper(
+            'feof',
+            $this->internalResource
+        );
+    }
+
+    /**
+     * @return bool
+     */
+    public function stream_flush()
+    {
+        return $this->invokeInternalStreamWrapper(
+            'fflush',
+            $this->internalResource
+        );
+    }
+
+    /**
+     * @param int $operation
+     * @return bool
+     */
+    public function stream_lock($operation)
+    {
+        return $this->invokeInternalStreamWrapper(
+            'flock',
+            $this->internalResource,
+            $operation
+        );
+    }
+
+    /**
+     * @param string $path
+     * @param int $option
+     * @param string|int $value
+     * @return bool
+     */
+    public function stream_metadata($path, $option, $value)
+    {
+        $this->assert($path, Behavior::COMMAND_STEAM_METADATA);
+        if ($option === STREAM_META_TOUCH) {
+            return call_user_func_array(
+                array($this, 'invokeInternalStreamWrapper'),
+                array_merge(array('touch', $path), (array) $value)
+            );
+        }
+        if ($option === STREAM_META_OWNER_NAME || $option === STREAM_META_OWNER) {
+            return $this->invokeInternalStreamWrapper(
+                'chown',
+                $path,
+                $value
+            );
+        }
+        if ($option === STREAM_META_GROUP_NAME || $option === STREAM_META_GROUP) {
+            return $this->invokeInternalStreamWrapper(
+                'chgrp',
+                $path,
+                $value
+            );
+        }
+        if ($option === STREAM_META_ACCESS) {
+            return $this->invokeInternalStreamWrapper(
+                'chmod',
+                $path,
+                $value
+            );
+        }
+        return false;
+    }
+
+    /**
+     * @param string $path
+     * @param string $mode
+     * @param int $options
+     * @param string|null $opened_path
+     * @return bool
+     */
+    public function stream_open(
+        $path,
+        $mode,
+        $options,
+        &$opened_path = null
+    ) {
+        $this->assert($path, Behavior::COMMAND_STREAM_OPEN);
+        $arguments = array($path, $mode, (bool) ($options & STREAM_USE_PATH));
+        // only add stream context for non include/require calls
+        if (!($options & static::STREAM_OPEN_FOR_INCLUDE)) {
+            $arguments[] = $this->context;
+        // work around https://bugs.php.net/bug.php?id=66569
+        // for including files from Phar stream with OPcache enabled
+        } else {
+            Helper::resetOpCache();
+        }
+        $this->internalResource = call_user_func_array(
+            array($this, 'invokeInternalStreamWrapper'),
+            array_merge(array('fopen'), $arguments)
+        );
+        if (!is_resource($this->internalResource)) {
+            return false;
+        }
+        if ($opened_path !== null) {
+            $metaData = stream_get_meta_data($this->internalResource);
+            $opened_path = $metaData['uri'];
+        }
+        return true;
+    }
+
+    /**
+     * @param int $count
+     * @return string
+     */
+    public function stream_read($count)
+    {
+        return $this->invokeInternalStreamWrapper(
+            'fread',
+            $this->internalResource,
+            $count
+        );
+    }
+
+    /**
+     * @param int $offset
+     * @param int $whence
+     * @return bool
+     */
+    public function stream_seek($offset, $whence = SEEK_SET)
+    {
+        return $this->invokeInternalStreamWrapper(
+            'fseek',
+            $this->internalResource,
+            $offset,
+            $whence
+        ) !== -1;
+    }
+
+    /**
+     * @param int $option
+     * @param int $arg1
+     * @param int $arg2
+     * @return bool
+     */
+    public function stream_set_option($option, $arg1, $arg2)
+    {
+        if ($option === STREAM_OPTION_BLOCKING) {
+            return $this->invokeInternalStreamWrapper(
+                'stream_set_blocking',
+                $this->internalResource,
+                $arg1
+            );
+        }
+        if ($option === STREAM_OPTION_READ_TIMEOUT) {
+            return $this->invokeInternalStreamWrapper(
+                'stream_set_timeout',
+                $this->internalResource,
+                $arg1,
+                $arg2
+            );
+        }
+        if ($option === STREAM_OPTION_WRITE_BUFFER) {
+            return $this->invokeInternalStreamWrapper(
+                'stream_set_write_buffer',
+                $this->internalResource,
+                $arg2
+            ) === 0;
+        }
+        return false;
+    }
+
+    /**
+     * @return array
+     */
+    public function stream_stat()
+    {
+        return $this->invokeInternalStreamWrapper(
+            'fstat',
+            $this->internalResource
+        );
+    }
+
+    /**
+     * @return int
+     */
+    public function stream_tell()
+    {
+        return $this->invokeInternalStreamWrapper(
+            'ftell',
+            $this->internalResource
+        );
+    }
+
+    /**
+     * @param int $new_size
+     * @return bool
+     */
+    public function stream_truncate($new_size)
+    {
+        return $this->invokeInternalStreamWrapper(
+            'ftruncate',
+            $this->internalResource,
+            $new_size
+        );
+    }
+
+    /**
+     * @param string $data
+     * @return int
+     */
+    public function stream_write($data)
+    {
+        return $this->invokeInternalStreamWrapper(
+            'fwrite',
+            $this->internalResource,
+            $data
+        );
+    }
+
+    /**
+     * @param string $path
+     * @return bool
+     */
+    public function unlink($path)
+    {
+        $this->assert($path, Behavior::COMMAND_UNLINK);
+        return $this->invokeInternalStreamWrapper(
+            'unlink',
+            $path,
+            $this->context
+        );
+    }
+
+    /**
+     * @param string $path
+     * @param int $flags
+     * @return array|false
+     */
+    public function url_stat($path, $flags)
+    {
+        $this->assert($path, Behavior::COMMAND_URL_STAT);
+        $functionName = $flags & STREAM_URL_STAT_QUIET ? '@stat' : 'stat';
+        return $this->invokeInternalStreamWrapper($functionName, $path);
+    }
+
+    /**
+     * @param string $path
+     * @param string $command
+     */
+    protected function assert($path, $command)
+    {
+        if (Manager::instance()->assert($path, $command) === true) {
+            $this->collectInvocation($path);
+            return;
+        }
+
+        throw new Exception(
+            sprintf(
+                'Denied invocation of "%s" for command "%s"',
+                $path,
+                $command
+            ),
+            1535189880
+        );
+    }
+
+    /**
+     * @param string $path
+     */
+    protected function collectInvocation($path)
+    {
+        if (isset($this->invocation)) {
+            return;
+        }
+
+        $manager = Manager::instance();
+        $this->invocation = $manager->resolve($path);
+        if ($this->invocation === null) {
+            throw new Exception(
+                'Expected invocation could not be resolved',
+                1556389591
+            );
+        }
+        // confirm, previous interceptor(s) validated invocation
+        $this->invocation->confirm();
+        $collection = $manager->getCollection();
+        if (!$collection->has($this->invocation)) {
+            $collection->collect($this->invocation);
+        }
+    }
+
+    /**
+     * @return Manager|Assertable
+     * @deprecated Use Manager::instance() directly
+     */
+    protected function resolveAssertable()
+    {
+        return Manager::instance();
+    }
+
+    /**
+     * Invokes commands on the native PHP Phar stream wrapper.
+     *
+     * @param string $functionName
+     * @param mixed ...$arguments
+     * @return mixed
+     */
+    private function invokeInternalStreamWrapper($functionName)
+    {
+        $arguments = func_get_args();
+        array_shift($arguments);
+        $silentExecution = $functionName[0] === '@';
+        $functionName = ltrim($functionName, '@');
+        $this->restoreInternalSteamWrapper();
+
+        try {
+            if ($silentExecution) {
+                $result = @call_user_func_array($functionName, $arguments);
+            } else {
+                $result = call_user_func_array($functionName, $arguments);
+            }
+        } catch (\Exception $exception) {
+            $this->registerStreamWrapper();
+            throw $exception;
+        } catch (\Throwable $throwable) {
+            $this->registerStreamWrapper();
+            throw $throwable;
+        }
+
+        $this->registerStreamWrapper();
+        return $result;
+    }
+
+    private function restoreInternalSteamWrapper()
+    {
+        stream_wrapper_restore('phar');
+    }
+
+    private function registerStreamWrapper()
+    {
+        stream_wrapper_unregister('phar');
+        stream_wrapper_register('phar', get_class($this));
+    }
+}

misc/typo3/phar-stream-wrapper/src/Resolvable.php

@@ -0,0 +1,24 @@
+<?php
+namespace TYPO3\PharStreamWrapper;
+
+/*
+ * This file is part of the TYPO3 project.
+ *
+ * It is free software; you can redistribute it and/or modify it under the terms
+ * of the MIT License (MIT). For the full copyright and license information,
+ * please read the LICENSE file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use TYPO3\PharStreamWrapper\Resolver\PharInvocation;
+
+interface Resolvable
+{
+    /**
+     * @param string $path
+     * @param null|int $flags
+     * @return null|PharInvocation
+     */
+    public function resolve($path, $flags = null);
+}