1 /*

     2     Copyright (C) 1998 Lars Knoll (knoll@mpi-hd.mpg.de)

     3     Copyright (C) 2001 Dirk Mueller (mueller@kde.org)

     4     Copyright (C) 2002 Waldo Bastian (bastian@kde.org)

     5     Copyright (C) 2006 Samuel Weinig (sam.weinig@gmail.com)

     6     Copyright (C) 2004, 2005, 2006, 2007, 2008 Apple Inc. All rights reserved.

     7 

     8     This library is free software; you can redistribute it and/or

     9     modify it under the terms of the GNU Library General Public

    10     License as published by the Free Software Foundation; either

    11     version 2 of the License, or (at your option) any later version.

    12 

    13     This library is distributed in the hope that it will be useful,

    14     but WITHOUT ANY WARRANTY; without even the implied warranty of

    15     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

    16     Library General Public License for more details.

    17 

    18     You should have received a copy of the GNU Library General Public License

    19     along with this library; see the file COPYING.LIB.  If not, write to

    20     the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,

    21     Boston, MA 02110-1301, USA.

    22 */

    23 

    24 #include "config.h"

    25 #include "loader.h"

    26 

    27 #include "Cache.h"

    28 #include "CachedImage.h"

    29 #include "CachedResource.h"

    30 #include "DocLoader.h"

    31 #include "InspectorTimelineAgent.h"

    32 #include "Frame.h"

    33 #include "FrameLoader.h"

    34 #include "HTMLDocument.h"

    35 #include "Request.h"

    36 #include "ResourceHandle.h"

    37 #include "ResourceRequest.h"

    38 #include "ResourceResponse.h"

    39 #include "SecurityOrigin.h"

    40 #include "SharedBuffer.h"

    41 #include "SubresourceLoader.h"

    42 #include <wtf/Assertions.h>

    43 #include <wtf/Vector.h>

    44 

    45 #define REQUEST_MANAGEMENT_ENABLED 1

    46 #define REQUEST_DEBUG 0

    47 

    48 namespace WebCore {

    49 

    50 #if REQUEST_MANAGEMENT_ENABLED

    51 // Match the parallel connection count used by the networking layer

    52 static unsigned maxRequestsInFlightPerHost;

    53 // Having a limit might still help getting more important resources first

    54 static const unsigned maxRequestsInFlightForNonHTTPProtocols = 20;

    55 #else

    56 static const unsigned maxRequestsInFlightPerHost = 10000;

    57 static const unsigned maxRequestsInFlightForNonHTTPProtocols = 10000;

    58 #endif

    59 

    60 Loader::Loader()

    61     : m_requestTimer(this, &Loader::requestTimerFired)

    62     , m_isSuspendingPendingRequests(false)

    63 {

    64     m_nonHTTPProtocolHost = Host::create(AtomicString(), maxRequestsInFlightForNonHTTPProtocols);

    65 #if REQUEST_MANAGEMENT_ENABLED

    66     maxRequestsInFlightPerHost = initializeMaximumHTTPConnectionCountPerHost();

    67 #endif

    68 }

    69 

    70 Loader::~Loader()

    71 {    

    72     ASSERT_NOT_REACHED();

    73 }

    74     

    75 static ResourceRequest::TargetType cachedResourceTypeToTargetType(CachedResource::Type type)

    76 {

    77     switch (type) {

    78     case CachedResource::CSSStyleSheet:

    79 #if ENABLE(XSLT)

    80     case CachedResource::XSLStyleSheet:

    81 #endif

    82 #if ENABLE(XBL)

    83     case CachedResource::XBL:

    84 #endif

    85         return ResourceRequest::TargetIsStyleSheet;

    86     case CachedResource::Script: 

    87         return ResourceRequest::TargetIsScript;

    88     case CachedResource::FontResource:

    89         return ResourceRequest::TargetIsFontResource;

    90     case CachedResource::ImageResource:

    91         return ResourceRequest::TargetIsImage;

    92 #if ENABLE(LINK_PREFETCH)

    93     case CachedResource::LinkPrefetch:

    94         return ResourceRequest::TargetIsPrefetch;

    95 #endif

    96     }

    97     ASSERT_NOT_REACHED();

    98     return ResourceRequest::TargetIsSubresource;

    99 }

   100 

   101 Loader::Priority Loader::determinePriority(const CachedResource* resource) const

   102 {

   103 #if REQUEST_MANAGEMENT_ENABLED

   104     switch (resource->type()) {

   105         case CachedResource::CSSStyleSheet:

   106 #if ENABLE(XSLT)

   107         case CachedResource::XSLStyleSheet:

   108 #endif

   109 #if ENABLE(XBL)

   110         case CachedResource::XBL:

   111 #endif

   112             return High;

   113         case CachedResource::Script: 

   114         case CachedResource::FontResource:

   115             return Medium;

   116         case CachedResource::ImageResource:

   117             return Low;

   118 #if ENABLE(LINK_PREFETCH)

   119         case CachedResource::LinkPrefetch:

   120             return VeryLow;

   121 #endif

   122     }

   123     ASSERT_NOT_REACHED();

   124     return Low;

   125 #else

   126     return High;

   127 #endif

   128 }

   129 

   130 void Loader::load(DocLoader* docLoader, CachedResource* resource, bool incremental, SecurityCheckPolicy securityCheck, bool sendResourceLoadCallbacks)

   131 {

   132     ASSERT(docLoader);

   133     Request* request = new Request(docLoader, resource, incremental, securityCheck, sendResourceLoadCallbacks);

   134 

   135     RefPtr<Host> host;

   136     KURL url(ParsedURLString, resource->url());

   137     if (url.protocolInHTTPFamily()) {

   138         m_hosts.checkConsistency();

   139         AtomicString hostName = url.host();

   140         host = m_hosts.get(hostName.impl());

   141         if (!host) {

   142             host = Host::create(hostName, maxRequestsInFlightPerHost);

   143             m_hosts.add(hostName.impl(), host);

   144         }

   145     } else 

   146         host = m_nonHTTPProtocolHost;

   147     

   148     bool hadRequests = host->hasRequests();

   149     Priority priority = determinePriority(resource);

   150     host->addRequest(request, priority);

   151     docLoader->incrementRequestCount(request->cachedResource());

   152 

   153     if (priority > Low || !url.protocolInHTTPFamily() || (priority == Low && !hadRequests)) {

   154         // Try to request important resources immediately

   155         host->servePendingRequests(priority);

   156     } else {

   157         // Handle asynchronously so early low priority requests don't get scheduled before later high priority ones

   158 #if ENABLE(INSPECTOR)

   159         if (InspectorTimelineAgent::instanceCount()) {

   160             InspectorTimelineAgent* agent = docLoader->doc()->inspectorTimelineAgent();

   161             if (agent)

   162                 agent->didScheduleResourceRequest(resource->url());

   163         }

   164 #endif // ENABLE(INSPECTOR)

   165         scheduleServePendingRequests();

   166     }

   167 }

   168     

   169 void Loader::scheduleServePendingRequests()

   170 {

   171     if (!m_requestTimer.isActive())

   172         m_requestTimer.startOneShot(0);

   173 }

   174 

   175 void Loader::requestTimerFired(Timer<Loader>*) 

   176 {

   177     servePendingRequests();

   178 }

   179 

   180 void Loader::servePendingRequests(Priority minimumPriority)

   181 {

   182     if (m_isSuspendingPendingRequests)

   183         return;

   184 

   185     m_requestTimer.stop();

   186     

   187     m_nonHTTPProtocolHost->servePendingRequests(minimumPriority);

   188 

   189     Vector<Host*> hostsToServe;

   190     m_hosts.checkConsistency();

   191     HostMap::iterator i = m_hosts.begin();

   192     HostMap::iterator end = m_hosts.end();

   193     for (;i != end; ++i)

   194         hostsToServe.append(i->second.get());

   195         

   196     for (unsigned n = 0; n < hostsToServe.size(); ++n) {

   197         Host* host = hostsToServe[n];

   198         if (host->hasRequests())

   199             host->servePendingRequests(minimumPriority);

   200         else if (!host->processingResource()){

   201             AtomicString name = host->name();

   202             m_hosts.remove(name.impl());

   203         }

   204     }

   205 }

   206 

   207 void Loader::suspendPendingRequests()

   208 {

   209     ASSERT(!m_isSuspendingPendingRequests);

   210     m_isSuspendingPendingRequests = true;

   211 }

   212 

   213 void Loader::resumePendingRequests()

   214 {

   215     ASSERT(m_isSuspendingPendingRequests);

   216     m_isSuspendingPendingRequests = false;

   217     if (!m_hosts.isEmpty() || m_nonHTTPProtocolHost->hasRequests())

   218         scheduleServePendingRequests();

   219 }

   220 

   221 void Loader::nonCacheRequestInFlight(const KURL& url)

   222 {

   223     if (!url.protocolInHTTPFamily())

   224         return;

   225     

   226     AtomicString hostName = url.host();

   227     m_hosts.checkConsistency();

   228     RefPtr<Host> host = m_hosts.get(hostName.impl());

   229     if (!host) {

   230         host = Host::create(hostName, maxRequestsInFlightPerHost);

   231         m_hosts.add(hostName.impl(), host);

   232     }

   233 

   234     host->nonCacheRequestInFlight();

   235 }

   236 

   237 void Loader::nonCacheRequestComplete(const KURL& url)

   238 {

   239     if (!url.protocolInHTTPFamily())

   240         return;

   241     

   242     AtomicString hostName = url.host();

   243     m_hosts.checkConsistency();

   244     RefPtr<Host> host = m_hosts.get(hostName.impl());

   245     ASSERT(host);

   246     if (!host)

   247         return;

   248 

   249     host->nonCacheRequestComplete();

   250 }

   251 

   252 void Loader::cancelRequests(DocLoader* docLoader)

   253 {

   254     docLoader->clearPendingPreloads();

   255 

   256     if (m_nonHTTPProtocolHost->hasRequests())

   257         m_nonHTTPProtocolHost->cancelRequests(docLoader);

   258     

   259     Vector<Host*> hostsToCancel;

   260     m_hosts.checkConsistency();

   261     HostMap::iterator i = m_hosts.begin();

   262     HostMap::iterator end = m_hosts.end();

   263     for (;i != end; ++i)

   264         hostsToCancel.append(i->second.get());

   265 

   266     for (unsigned n = 0; n < hostsToCancel.size(); ++n) {

   267         Host* host = hostsToCancel[n];

   268         if (host->hasRequests())

   269             host->cancelRequests(docLoader);

   270     }

   271 

   272     scheduleServePendingRequests();

   273     

   274     ASSERT(docLoader->requestCount() == (docLoader->loadInProgress() ? 1 : 0));

   275 }

   276 

   277 Loader::Host::Host(const AtomicString& name, unsigned maxRequestsInFlight)

   278     : m_name(name)

   279     , m_maxRequestsInFlight(maxRequestsInFlight)

   280     , m_numResourcesProcessing(0)

   281     , m_nonCachedRequestsInFlight(0)

   282 {

   283 }

   284 

   285 Loader::Host::~Host()

   286 {

   287     ASSERT(m_requestsLoading.isEmpty());

   288     for (unsigned p = 0; p <= High; p++)

   289         ASSERT(m_requestsPending[p].isEmpty());

   290 }

   291     

   292 void Loader::Host::addRequest(Request* request, Priority priority)

   293 {

   294     m_requestsPending[priority].append(request);

   295 }

   296     

   297 void Loader::Host::nonCacheRequestInFlight()

   298 {

   299     ++m_nonCachedRequestsInFlight;

   300 }

   301 

   302 void Loader::Host::nonCacheRequestComplete()

   303 {

   304     --m_nonCachedRequestsInFlight;

   305     ASSERT(m_nonCachedRequestsInFlight >= 0);

   306 

   307     cache()->loader()->scheduleServePendingRequests();

   308 }

   309 

   310 bool Loader::Host::hasRequests() const

   311 {

   312     if (!m_requestsLoading.isEmpty())

   313         return true;

   314     for (unsigned p = 0; p <= High; p++) {

   315         if (!m_requestsPending[p].isEmpty())

   316             return true;

   317     }

   318     return false;

   319 }

   320 

   321 void Loader::Host::servePendingRequests(Loader::Priority minimumPriority)

   322 {

   323     if (cache()->loader()->isSuspendingPendingRequests())

   324         return;

   325 

   326     bool serveMore = true;

   327     for (int priority = High; priority >= minimumPriority && serveMore; --priority)

   328         servePendingRequests(m_requestsPending[priority], serveMore);

   329 }

   330 

   331 void Loader::Host::servePendingRequests(RequestQueue& requestsPending, bool& serveLowerPriority)

   332 {

   333     while (!requestsPending.isEmpty()) {        

   334         Request* request = requestsPending.first();

   335         DocLoader* docLoader = request->docLoader();

   336         bool resourceIsCacheValidator = request->cachedResource()->isCacheValidator();

   337 

   338         // For named hosts - which are only http(s) hosts - we should always enforce the connection limit.

   339         // For non-named hosts - everything but http(s) - we should only enforce the limit if the document isn't done parsing 

   340         // and we don't know all stylesheets yet.

   341         bool shouldLimitRequests = !m_name.isNull() || docLoader->doc()->parsing() || !docLoader->doc()->haveStylesheetsLoaded();

   342         if (shouldLimitRequests && m_requestsLoading.size() + m_nonCachedRequestsInFlight >= m_maxRequestsInFlight) {

   343             serveLowerPriority = false;

   344             return;

   345         }

   346         requestsPending.removeFirst();

   347         

   348         ResourceRequest resourceRequest(request->cachedResource()->url());

   349         resourceRequest.setTargetType(cachedResourceTypeToTargetType(request->cachedResource()->type()));

   350         

   351         if (!request->cachedResource()->accept().isEmpty())

   352             resourceRequest.setHTTPAccept(request->cachedResource()->accept());

   353         

   354          // Do not set the referrer or HTTP origin here. That's handled by SubresourceLoader::create.

   355         

   356         if (resourceIsCacheValidator) {

   357             CachedResource* resourceToRevalidate = request->cachedResource()->resourceToRevalidate();

   358             ASSERT(resourceToRevalidate->canUseCacheValidator());

   359             ASSERT(resourceToRevalidate->isLoaded());

   360             const String& lastModified = resourceToRevalidate->response().httpHeaderField("Last-Modified");

   361             const String& eTag = resourceToRevalidate->response().httpHeaderField("ETag");

   362             if (!lastModified.isEmpty() || !eTag.isEmpty()) {

   363                 ASSERT(docLoader->cachePolicy() != CachePolicyReload);

   364                 if (docLoader->cachePolicy() == CachePolicyRevalidate)

   365                     resourceRequest.setHTTPHeaderField("Cache-Control", "max-age=0");

   366                 if (!lastModified.isEmpty())

   367                     resourceRequest.setHTTPHeaderField("If-Modified-Since", lastModified);

   368                 if (!eTag.isEmpty())

   369                     resourceRequest.setHTTPHeaderField("If-None-Match", eTag);

   370             }

   371         }

   372 

   373         RefPtr<SubresourceLoader> loader = SubresourceLoader::create(docLoader->doc()->frame(),

   374             this, resourceRequest, request->shouldDoSecurityCheck(), request->sendResourceLoadCallbacks());

   375         if (loader) {

   376             m_requestsLoading.add(loader.release(), request);

   377             request->cachedResource()->setRequestedFromNetworkingLayer();

   378 #if REQUEST_DEBUG

   379             printf("HOST %s COUNT %d LOADING %s\n", resourceRequest.url().host().latin1().data(), m_requestsLoading.size(), request->cachedResource()->url().latin1().data());

   380 #endif

   381         } else {            

   382             docLoader->decrementRequestCount(request->cachedResource());

   383             docLoader->setLoadInProgress(true);

   384             request->cachedResource()->error();

   385             docLoader->setLoadInProgress(false);

   386             delete request;

   387         }

   388     }

   389 }

   390 

   391 void Loader::Host::didFinishLoading(SubresourceLoader* loader)

   392 {

   393     RefPtr<Host> myProtector(this);

   394 

   395     RequestMap::iterator i = m_requestsLoading.find(loader);

   396     if (i == m_requestsLoading.end())

   397         return;

   398     

   399     Request* request = i->second;

   400     m_requestsLoading.remove(i);

   401     DocLoader* docLoader = request->docLoader();

   402     // Prevent the document from being destroyed before we are done with

   403     // the docLoader that it will delete when the document gets deleted.

   404     RefPtr<Document> protector(docLoader->doc());

   405     if (!request->isMultipart())

   406         docLoader->decrementRequestCount(request->cachedResource());

   407 

   408     CachedResource* resource = request->cachedResource();

   409     ASSERT(!resource->resourceToRevalidate());

   410 

   411     // If we got a 4xx response, we're pretending to have received a network

   412     // error, so we can't send the successful data() and finish() callbacks.

   413     if (!resource->errorOccurred()) {

   414         docLoader->setLoadInProgress(true);

   415         resource->data(loader->resourceData(), true);

   416         resource->finish();

   417     }

   418 

   419     delete request;

   420 

   421     docLoader->setLoadInProgress(false);

   422     

   423     docLoader->checkForPendingPreloads();

   424 

   425 #if REQUEST_DEBUG

   426     KURL u(ParsedURLString, resource->url());

   427     printf("HOST %s COUNT %d RECEIVED %s\n", u.host().latin1().data(), m_requestsLoading.size(), resource->url().latin1().data());

   428 #endif

   429     servePendingRequests();

   430 }

   431 

   432 void Loader::Host::didFail(SubresourceLoader* loader, const ResourceError&)

   433 {

   434     didFail(loader);

   435 }

   436 

   437 void Loader::Host::didFail(SubresourceLoader* loader, bool cancelled)

   438 {

   439     RefPtr<Host> myProtector(this);

   440 

   441     loader->clearClient();

   442 

   443     RequestMap::iterator i = m_requestsLoading.find(loader);

   444     if (i == m_requestsLoading.end())

   445         return;

   446     

   447     Request* request = i->second;

   448     m_requestsLoading.remove(i);

   449     DocLoader* docLoader = request->docLoader();

   450     // Prevent the document from being destroyed before we are done with

   451     // the docLoader that it will delete when the document gets deleted.

   452     RefPtr<Document> protector(docLoader->doc());

   453     if (!request->isMultipart())

   454         docLoader->decrementRequestCount(request->cachedResource());

   455 

   456     CachedResource* resource = request->cachedResource();

   457     

   458     if (resource->resourceToRevalidate())

   459         cache()->revalidationFailed(resource);

   460 

   461     if (!cancelled) {

   462         docLoader->setLoadInProgress(true);

   463         resource->error();

   464     }

   465     

   466     docLoader->setLoadInProgress(false);

   467     if (cancelled || !resource->isPreloaded())

   468         cache()->remove(resource);

   469     

   470     delete request;

   471     

   472     docLoader->checkForPendingPreloads();

   473 

   474     servePendingRequests();

   475 }

   476 

   477 void Loader::Host::didReceiveResponse(SubresourceLoader* loader, const ResourceResponse& response)

   478 {

   479     RefPtr<Host> protector(this);

   480 

   481     Request* request = m_requestsLoading.get(loader);

   482     

   483     // FIXME: This is a workaround for <rdar://problem/5236843>

   484     // If a load starts while the frame is still in the provisional state 

   485     // (this can be the case when loading the user style sheet), committing the load then causes all

   486     // requests to be removed from the m_requestsLoading map. This means that request might be null here.

   487     // In that case we just return early. 

   488     // ASSERT(request);

   489     if (!request)

   490         return;

   491     

   492     CachedResource* resource = request->cachedResource();

   493     

   494     if (resource->isCacheValidator()) {

   495         if (response.httpStatusCode() == 304) {

   496             // 304 Not modified / Use local copy

   497             m_requestsLoading.remove(loader);

   498             loader->clearClient();

   499             request->docLoader()->decrementRequestCount(request->cachedResource());

   500 

   501             // Existing resource is ok, just use it updating the expiration time.

   502             cache()->revalidationSucceeded(resource, response);

   503             

   504             if (request->docLoader()->frame())

   505                 request->docLoader()->frame()->loader()->checkCompleted();

   506 

   507             delete request;

   508 

   509             servePendingRequests();

   510             return;

   511         } 

   512         // Did not get 304 response, continue as a regular resource load.

   513         cache()->revalidationFailed(resource);

   514     }

   515 

   516     resource->setResponse(response);

   517 

   518     String encoding = response.textEncodingName();

   519     if (!encoding.isNull())

   520         resource->setEncoding(encoding);

   521     

   522     if (request->isMultipart()) {

   523         ASSERT(resource->isImage());

   524         static_cast<CachedImage*>(resource)->clear();

   525         if (request->docLoader()->frame())

   526             request->docLoader()->frame()->loader()->checkCompleted();

   527     } else if (response.isMultipart()) {

   528         request->setIsMultipart(true);

   529         

   530         // We don't count multiParts in a DocLoader's request count

   531         request->docLoader()->decrementRequestCount(request->cachedResource());

   532 

   533         // If we get a multipart response, we must have a handle

   534         ASSERT(loader->handle());

   535         if (!resource->isImage())

   536             loader->handle()->cancel();

   537     }

   538 }

   539 

   540 void Loader::Host::didReceiveData(SubresourceLoader* loader, const char* data, int size)

   541 {

   542     RefPtr<Host> protector(this);

   543 

   544     Request* request = m_requestsLoading.get(loader);

   545     if (!request)

   546         return;

   547 

   548     CachedResource* resource = request->cachedResource();    

   549     ASSERT(!resource->isCacheValidator());

   550     

   551     if (resource->errorOccurred())

   552         return;

   553         

   554     if (resource->response().httpStatusCode() / 100 == 4) {

   555         // Treat a 4xx response like a network error for all resources but images (which will ignore the error and continue to load for 

   556         // legacy compatibility).

   557         resource->httpStatusCodeError();

   558         return;

   559     }

   560 

   561     // Set the data.

   562     if (request->isMultipart()) {

   563         // The loader delivers the data in a multipart section all at once, send eof.

   564         // The resource data will change as the next part is loaded, so we need to make a copy.

   565         RefPtr<SharedBuffer> copiedData = SharedBuffer::create(data, size);

   566         resource->data(copiedData.release(), true);

   567     } else if (request->isIncremental())

   568         resource->data(loader->resourceData(), false);

   569 }

   570     

   571 void Loader::Host::didReceiveCachedMetadata(SubresourceLoader* loader, const char* data, int size)

   572 {

   573     RefPtr<Host> protector(this);

   574 

   575     Request* request = m_requestsLoading.get(loader);

   576     if (!request)

   577         return;

   578 

   579     CachedResource* resource = request->cachedResource();    

   580     ASSERT(!resource->isCacheValidator());

   581 

   582     resource->setSerializedCachedMetadata(data, size);

   583 }

   584     

   585 void Loader::Host::cancelPendingRequests(RequestQueue& requestsPending, DocLoader* docLoader)

   586 {

   587     RequestQueue remaining;

   588     RequestQueue::iterator end = requestsPending.end();

   589     for (RequestQueue::iterator it = requestsPending.begin(); it != end; ++it) {

   590         Request* request = *it;

   591         if (request->docLoader() == docLoader) {

   592             cache()->remove(request->cachedResource());

   593             docLoader->decrementRequestCount(request->cachedResource());

   594             delete request;

   595         } else

   596             remaining.append(request);

   597     }

   598     requestsPending.swap(remaining);

   599 }

   600 

   601 void Loader::Host::cancelRequests(DocLoader* docLoader)

   602 {

   603     for (unsigned p = 0; p <= High; p++)

   604         cancelPendingRequests(m_requestsPending[p], docLoader);

   605 

   606     Vector<SubresourceLoader*, 256> loadersToCancel;

   607 

   608     RequestMap::iterator end = m_requestsLoading.end();

   609     for (RequestMap::iterator i = m_requestsLoading.begin(); i != end; ++i) {

   610         Request* r = i->second;

   611         if (r->docLoader() == docLoader)

   612             loadersToCancel.append(i->first.get());

   613     }

   614 

   615     for (unsigned i = 0; i < loadersToCancel.size(); ++i) {

   616         SubresourceLoader* loader = loadersToCancel[i];

   617         didFail(loader, true);

   618     }

   619 }

   620 

   621 } //namespace WebCore