How is Stack's imgur gallery usage distributed across the sites in the network?

Question

I just found out from this question (thanks, Shane!) that Stack's imgur namespace is still less than 1% allocated, with about 6.7M image IDs registered out of ~916B.

This naturally leads me to wonder: how is this image ID usage distributed across the sites in the network? I realize this is potentially a fuzzy question, since a given image could be linked from multiple sites in the network (or even from locations off the Stack network!) and thus "ownership" might be challenging to ascertain.

That said, does Stack and/or imgur, say, document the site through which a given image is uploaded? It seems like that would be a good proxy statistic.

Answer 1

I've tried a couple of approaches to answer your question.

• leveraging the StackApi search/advanced endpoint that offers and option to search for urls. Unfortunately that only searches in questions, not in answers.
• Using SEDE and query the posts table for body's that contain i.sstatic.net
• Using the on site search url:i.sstatic.net

StackApi

Using the /search/advanced endpoint for each site returns in the end that 584644 urls are used in questions. I didn't find a way to search in answers.

You can run the below Stack Snippet but keep in mind it only fires a search every 5 seconds and it needs to do all 163 sites and their meta's. There is some code in there that should prevent you from being throttled on your IP but I give you no guarantees. Run at your own risk.

var allsites = [],
    interval,
    SEApi,
    grandTotal;

// build api url for an endpoint and its optional parameters
function apiBuilder(endpoint, params) {
  var url = 'https://api.stackexchange.com/2.2/',
      urlPath = url + endpoint;
  params.key ='zHcti*qApcyvicd06oAFog((';
  if (params !== undefined)  {
    var query = [];
    for(var prop in params) {
      if (params.hasOwnProperty(prop)) {
        query.push( prop + '=' + encodeURI(params[prop]));
      }
    }
    urlPath = urlPath + '?' + query.join('&');
  }
  return urlPath;
}

function apiSearchBuilder(site, url) {
 return apiBuilder(
    'search/advanced', 
    {
      site: site,
      order: 'desc',
      url: url,
      sort: 'activity',
      filter: '!.UE8F0_EMkjm8QvG'
    });
}
    
// build url for /sites api endpoint
function apiSitesBuilder() {
  return apiBuilder(
    'sites', 
    {
      pagesize: 500,
      filter: '!2--Yion.3M.K5PJ6Ea0*R'
    });
}

// do a get on the API for the given url
// and invoke the callback with the JSON result
function API () {
  
  var backlog = [],
      getfunction;
  
  // simply push the params on the queue
  function cacheget(url, callback) {
    backlog.push({ url: url, callback: callback});
  }
  
  // this makes the actual xhr call
  function realget(url, callback) {
    var xhr = new XMLHttpRequest();
    
    // handles pending calls by invoking realget
    // and resetting the getfunction when 
    // the backlog is cleared
    function handleBacklog() {
      var item = backlog.shift();
      if (item !== undefined) {
        console.log('from cache');
        // handle this single item
        realget(item.url, item.callback);
      } 
      if (backlog.length === 0) {
        // if the backlog is empty 
        // use realget for the next call
        getfunction = realget;
      }
    }
    
    xhr.addEventListener('error', function () {
       console.log(xhr.status);
    });
    
    xhr.addEventListener('load', function () {
      var response = JSON.parse(xhr.responseText);
      var backoff = response.backoff || 0;
      // backoff received
      if (backoff > 0) {
        // start caching calls
        console.log('backoff recv');
        getfunction = cacheget;
      }
      if (response.error_id === 502) {
         console.log(reponse.error_message);
         getfunction = cacheget;
         backoff = 120; 
      }
      // process pending backlog 
      setTimeout(handleBacklog, backoff * 1000);
      // invoke the callback 
      callback(response);
    });
    xhr.open('GET', url);
    xhr.send();
  }
  
  // calls either xhr or the cache
  function get(url, callback)
  {
    getfunction(url, callback);
  }
  
  // initially we start with a realget
  getfunction = realget;
  
  // return the public api
  return {
    get: get
  }
}

SEApi = new API(); // keep an instance

function buildSiteRow(site, total) {
  var tr = document.createElement('tr'),
      tdsitename = document.createElement('td'),
      tdcount = document.createElement('td');
  tdsitename.textContent = site.name;
  tdcount.textContent = total;
  tr.appendChild(tdsitename);
  tr.appendChild(tdcount);
  return tr;
}

function processQueue() {
   var site = allsites.shift();
   if (site != undefined) {
      SEApi.get(apiSearchBuilder(site.api_site_parameter, '*//i.sstatic.net/*'), 
        function (result) {
          var tb = document
            .getElementById('result');
          tb.appendChild(buildSiteRow(site, result.total));
          grandTotal = grandTotal + result.total;
          tb.children[0].children[1].textContent = grandTotal;         
        }
      );
   } else {
      clearInterval(interval);
      var msg = document.getElementById('msg');
      msg.textContent = 'Done!';    
   }
}

// loadsites
// keep site records in allsites array
function  loadSites(search) {
  SEApi.get(apiSitesBuilder(), function(result) {
     var i;
      // loop over the items, being site types
      for(i = 0; i < result.items.length; i = i + 1) {
        // add site selection element
        allsites.push(result.items[i]);
      }
      grandTotal = 0;
      interval = setInterval(processQueue, 5000);
      search.textContent ='Processing sites ...'
   });
}

// hookup search button and start loadsites
function init() {
  var msg = document.getElementById('msg');
  msg.textContent = 'Loading ...';
  loadSites(msg);
}

// startup!
init();

table { border: solid black 1px;}
div {padding: 5px; }
th {font-style: bold; padding-bottom: 5px;}

<div id="msg">Start</div>
<table>
  <thead>
    <tr>
      <th>site name</th>
      <th># questions with imgur links</th>
    </tr>
  </thead>
  <tbody id="result">
    <tr><td>Total</td><td>0</td></tr>
  </tbody>
</table>

The Stack Exchange Data Explorer

Based on the comments with Cai I decided to give SEDE a beating as well.

Due to the sheer size of the data that needs to be searched and the time limit of 2 minutes for a query to return results I created a multi-database query that would search per year of post creation for the url. I saved the CSV file of each year and then used Excel fu to get the final results. You'll find the query here and here is a screenshot of the top result:

The raw dataset is here

As you can see this method gives a total of 1,923,034 posts with images.

As indicated by Shadow Wizard users can also store their profile image on imgur. This query does a cross-site query to count the number of userprofiles that have an link to imgur.

-- result table, don't rename and keep the site column
create table #results ( site nvarchar(250)
                       , ppcount int);

declare @sql nvarchar(max) = ''   -- holds build up sql string

-- build one biq union sql, for each db
select @sql = @sql 
+ iif( len(@sql) > 1 
     , 'union'
     , 'insert into #results'
) +
-- here goes the per site query, fully qualify the database objects
N'
select ''' + name + '''
     , count(*)
from '+  + quotename(name) + '.dbo.users 
where profileimageurl like ''%//i.sstatic.net%''
'
from sys.databases
where database_id > 5
-- and (name not like '%.Meta' or name = 'StackExchange.Meta')

--print @sql

-- execute it
exec (@sql)

select 'total'
     , sum(ppcount) as [# imgur profile pictures]
from #results
union
-- show results
select
       -- URL builder from Lamak http://stackoverflow.com/a/38129701
      'http://' 
      + case PARSENAME(site,1)
        when 'audio' then 'avp'
        else PARSENAME(site,1)
        end 
      + coalesce('.' 
        + case PARSENAME(site,2)
          when 'audio' then 'avp'
          else PARSENAME(site,2)
          end , '')
      + IIF(patindex('%.MathOverflow%',site) > 0
      , ''
      , coalesce('.' + PARSENAME(site,3), '')
      )
      + IIF(patindex('%.MathOverflow%',site) > 0
      , '.net'
      , '.com'
      ) 
      + '|'
      + replace(site, 'stackexchange.','')
      -- rest of columns
      , ppcount
from #results
order by 2 desc

drop table #results

When you run in this query today you'll find 1,658,892 links to the imgur domain of Stack Exchange.

A similar query can be run for the about me of the users. I leave that as an exercise for the reader as I don't expect to find another million.

Site Search

It would take a while to run the search url:i.sstatic.net on every site and I didn't bother scripting it. I did use it to verify my approach. I think this is scriptable in a browser so anyone who fancies that, be my guest.

Conclusion

None of the above methods will answer your question. First of all the applied methods only count posts (question or answers) that have an url to imgur in it. If a post uses 10 images, it would still count as one.

None of the methods above seem to agree on the number of posts they find. It is either the search or limitations of the provided api or data to get a consistent result.

What is most noticeable is the huge gap between my number 1,923,034 and the 6,713,997 from Shane. Maybe I can find a couple more in comments and the posthistory. And I didn't cover deleted posts. The only large chunk that is missing here are the images uploaded in chat. I doubt if that would bring 4 million images to the table.