Patchwork WIP: lookaside clone extension

login
register
mail settings
Submitter Augie Fackler
Date March 13, 2014, 2:58 a.m.
Message ID <c51e6e04d8c581e6d710.1394679536@augie-macbookair.alto.octopoid.net>
Download mbox | patch
Permalink /patch/3934/
State Rejected
Headers show

Comments

Augie Fackler - March 13, 2014, 2:58 a.m.
# HG changeset patch
# User Augie Fackler <raf@durin42.com>
# Date 1380062977 14400
#      Tue Sep 24 18:49:37 2013 -0400
# Node ID c51e6e04d8c581e6d7109fb3a54a69121c12d7e5
# Parent  1cd5bff45db28150d7c140be493fe851e6560f27
WIP: lookaside clone extension
Augie Fackler - March 13, 2014, 3:01 a.m.
Messed up a for loop in my shell, drop this one for now please.

On Wed, Mar 12, 2014 at 7:58 PM, Augie Fackler <raf@durin42.com> wrote:

> # HG changeset patch
> # User Augie Fackler <raf@durin42.com>
> # Date 1380062977 14400
> #      Tue Sep 24 18:49:37 2013 -0400
> # Node ID c51e6e04d8c581e6d7109fb3a54a69121c12d7e5
> # Parent  1cd5bff45db28150d7c140be493fe851e6560f27
> WIP: lookaside clone extension
>
> diff --git a/hgext/lookaside.py b/hgext/lookaside.py
> new file mode 100644
> --- /dev/null
> +++ b/hgext/lookaside.py
> @@ -0,0 +1,53 @@
> +"""Hacky proof of concept for lookaside bundle.
> +
> +Right now this doesn't do anything clever about bundle generation or
> +distribution. Put a bundle file (generated with 'hg bundle --all' at a
> +URL that urllib2 can fetch, then put that url in
> +.hg/durin42-lookaside-location.
> +
> +This might mulch your data somehow. It's relatively untested, but it
> +proves the concept.
> +"""
> +
> +import shutil
> +import urllib2
> +
> +from mercurial import changegroup
> +from mercurial import extensions
> +from mercurial import wireproto
> +
> +def addcap(orig, *args, **kwargs):
> +    c = orig(*args, **kwargs)
> +    return c + ' x-durin42-lookaside'
> +
> +extensions.wrapfunction(wireproto, 'capabilities', addcap)
> +wireproto.commands['capabilities'] = wireproto.capabilities, ''
> +
> +def lookaside(repo, proto):
> +    try:
> +        with repo.opener('durin42-lookaside-location') as f:
> +            return f.read().strip()
> +    except IOError:
> +        return ''
> +
> +
> +wireproto.commands['x-durin42-lookaside-clone'] = lookaside, ''
> +
> +def reposetup(ui, repo):
> +    origcls = repo.__class__
> +
> +    class lookasiderepo(origcls):
> +
> +        def clone(self, remote, heads=[], stream=False):
> +            if not heads and remote.capable('x-durin42-lookaside'):
> +                url = remote._call('x-durin42-lookaside-clone')
> +                if url:
> +                    ui.debug('fetching lookaside bundle %s\n' % url)
> +                    # TODO: support resuming the bundle stream if a
> connection fails
> +                    self.addchangegroup(
> +                        changegroup.readbundle(urllib2.urlopen(url),
> 'lookaside bundle'),
> +                        'unbundle', 'bundle:lookaside bundle')
> +                return self.pull(remote, heads)
> +            return origcls.clone(self, remote, heads, stream=False)
> +
> +    repo.__class__ = lookasiderepo
> _______________________________________________
> Mercurial-devel mailing list
> Mercurial-devel@selenic.com
> http://selenic.com/mailman/listinfo/mercurial-devel
>
Augie Fackler - March 13, 2014, 3:02 a.m.
And marmoute just gave me a great idea: we can do this using bundle2, by
having the extension produce a header in the bundle that says "go fetch
this URL, apply it, and then apply the stuff I gave you"


On Wed, Mar 12, 2014 at 8:01 PM, Augie Fackler <raf@durin42.com> wrote:

> Messed up a for loop in my shell, drop this one for now please.
>
>
> On Wed, Mar 12, 2014 at 7:58 PM, Augie Fackler <raf@durin42.com> wrote:
>
>> # HG changeset patch
>> # User Augie Fackler <raf@durin42.com>
>> # Date 1380062977 14400
>> #      Tue Sep 24 18:49:37 2013 -0400
>> # Node ID c51e6e04d8c581e6d7109fb3a54a69121c12d7e5
>> # Parent  1cd5bff45db28150d7c140be493fe851e6560f27
>> WIP: lookaside clone extension
>>
>> diff --git a/hgext/lookaside.py b/hgext/lookaside.py
>> new file mode 100644
>> --- /dev/null
>> +++ b/hgext/lookaside.py
>> @@ -0,0 +1,53 @@
>> +"""Hacky proof of concept for lookaside bundle.
>> +
>> +Right now this doesn't do anything clever about bundle generation or
>> +distribution. Put a bundle file (generated with 'hg bundle --all' at a
>> +URL that urllib2 can fetch, then put that url in
>> +.hg/durin42-lookaside-location.
>> +
>> +This might mulch your data somehow. It's relatively untested, but it
>> +proves the concept.
>> +"""
>> +
>> +import shutil
>> +import urllib2
>> +
>> +from mercurial import changegroup
>> +from mercurial import extensions
>> +from mercurial import wireproto
>> +
>> +def addcap(orig, *args, **kwargs):
>> +    c = orig(*args, **kwargs)
>> +    return c + ' x-durin42-lookaside'
>> +
>> +extensions.wrapfunction(wireproto, 'capabilities', addcap)
>> +wireproto.commands['capabilities'] = wireproto.capabilities, ''
>> +
>> +def lookaside(repo, proto):
>> +    try:
>> +        with repo.opener('durin42-lookaside-location') as f:
>> +            return f.read().strip()
>> +    except IOError:
>> +        return ''
>> +
>> +
>> +wireproto.commands['x-durin42-lookaside-clone'] = lookaside, ''
>> +
>> +def reposetup(ui, repo):
>> +    origcls = repo.__class__
>> +
>> +    class lookasiderepo(origcls):
>> +
>> +        def clone(self, remote, heads=[], stream=False):
>> +            if not heads and remote.capable('x-durin42-lookaside'):
>> +                url = remote._call('x-durin42-lookaside-clone')
>> +                if url:
>> +                    ui.debug('fetching lookaside bundle %s\n' % url)
>> +                    # TODO: support resuming the bundle stream if a
>> connection fails
>> +                    self.addchangegroup(
>> +                        changegroup.readbundle(urllib2.urlopen(url),
>> 'lookaside bundle'),
>> +                        'unbundle', 'bundle:lookaside bundle')
>> +                return self.pull(remote, heads)
>> +            return origcls.clone(self, remote, heads, stream=False)
>> +
>> +    repo.__class__ = lookasiderepo
>> _______________________________________________
>> Mercurial-devel mailing list
>> Mercurial-devel@selenic.com
>> http://selenic.com/mailman/listinfo/mercurial-devel
>>
>
>
Gregory Szorc - July 19, 2014, 8 p.m.
I took the idea of this extension and built it out into a more finished 
implementation, which I'm calling "bundleclone."

https://hg.mozilla.org/hgcustom/version-control-tools/file/default/hgext/bundleclone

We may deploy this at Mozilla to reduce server load. More details at 
https://bugzilla.mozilla.org/show_bug.cgi?id=1041173

I'd appreciate any feedback.

On 3/12/14, 9:58 PM, Augie Fackler wrote:
> # HG changeset patch
> # User Augie Fackler <raf@durin42.com>
> # Date 1380062977 14400
> #      Tue Sep 24 18:49:37 2013 -0400
> # Node ID c51e6e04d8c581e6d7109fb3a54a69121c12d7e5
> # Parent  1cd5bff45db28150d7c140be493fe851e6560f27
> WIP: lookaside clone extension
>
> diff --git a/hgext/lookaside.py b/hgext/lookaside.py
> new file mode 100644
> --- /dev/null
> +++ b/hgext/lookaside.py
> @@ -0,0 +1,53 @@
> +"""Hacky proof of concept for lookaside bundle.
> +
> +Right now this doesn't do anything clever about bundle generation or
> +distribution. Put a bundle file (generated with 'hg bundle --all' at a
> +URL that urllib2 can fetch, then put that url in
> +.hg/durin42-lookaside-location.
> +
> +This might mulch your data somehow. It's relatively untested, but it
> +proves the concept.
> +"""
> +
> +import shutil
> +import urllib2
> +
> +from mercurial import changegroup
> +from mercurial import extensions
> +from mercurial import wireproto
> +
> +def addcap(orig, *args, **kwargs):
> +    c = orig(*args, **kwargs)
> +    return c + ' x-durin42-lookaside'
> +
> +extensions.wrapfunction(wireproto, 'capabilities', addcap)
> +wireproto.commands['capabilities'] = wireproto.capabilities, ''
> +
> +def lookaside(repo, proto):
> +    try:
> +        with repo.opener('durin42-lookaside-location') as f:
> +            return f.read().strip()
> +    except IOError:
> +        return ''
> +
> +
> +wireproto.commands['x-durin42-lookaside-clone'] = lookaside, ''
> +
> +def reposetup(ui, repo):
> +    origcls = repo.__class__
> +
> +    class lookasiderepo(origcls):
> +
> +        def clone(self, remote, heads=[], stream=False):
> +            if not heads and remote.capable('x-durin42-lookaside'):
> +                url = remote._call('x-durin42-lookaside-clone')
> +                if url:
> +                    ui.debug('fetching lookaside bundle %s\n' % url)
> +                    # TODO: support resuming the bundle stream if a connection fails
> +                    self.addchangegroup(
> +                        changegroup.readbundle(urllib2.urlopen(url), 'lookaside bundle'),
> +                        'unbundle', 'bundle:lookaside bundle')
> +                return self.pull(remote, heads)
> +            return origcls.clone(self, remote, heads, stream=False)
> +
> +    repo.__class__ = lookasiderepo
> _______________________________________________
> Mercurial-devel mailing list
> Mercurial-devel@selenic.com
> http://selenic.com/mailman/listinfo/mercurial-devel
>
Matt Mackall - July 19, 2014, 8:25 p.m.
On Sat, 2014-07-19 at 15:00 -0500, Gregory Szorc wrote:
> I took the idea of this extension and built it out into a more finished 
> implementation, which I'm calling "bundleclone."
> 
> https://hg.mozilla.org/hgcustom/version-control-tools/file/default/hgext/bundleclone
> 
> We may deploy this at Mozilla to reduce server load. More details at 
> https://bugzilla.mozilla.org/show_bug.cgi?id=1041173
> 
> I'd appreciate any feedback.

Can I get you to make a wiki page for it?
Matt Mackall - July 19, 2014, 9:06 p.m.
On Sat, 2014-07-19 at 15:00 -0500, Gregory Szorc wrote:
> I took the idea of this extension and built it out into a more finished 
> implementation, which I'm calling "bundleclone."
> 
> https://hg.mozilla.org/hgcustom/version-control-tools/file/default/hgext/bundleclone
> 
> We may deploy this at Mozilla to reduce server load. More details at 
> https://bugzilla.mozilla.org/show_bug.cgi?id=1041173
> 
> I'd appreciate any feedback.

Can I get you to make a wiki page for it?
Gregory Szorc - July 21, 2014, 12:30 a.m.
On 7/19/14, 3:25 PM, Matt Mackall wrote:
> On Sat, 2014-07-19 at 15:00 -0500, Gregory Szorc wrote:
>> I took the idea of this extension and built it out into a more finished
>> implementation, which I'm calling "bundleclone."
>>
>> https://hg.mozilla.org/hgcustom/version-control-tools/file/default/hgext/bundleclone
>>
>> We may deploy this at Mozilla to reduce server load. More details at
>> https://bugzilla.mozilla.org/show_bug.cgi?id=1041173
>>
>> I'd appreciate any feedback.
>
> Can I get you to make a wiki page for it?

http://mercurial.selenic.com/wiki/BundleCloneExtension

I also backported the extension to work with 2.5 and above.
Augie Fackler - July 21, 2014, 1:42 p.m.
On Jul 20, 2014, at 8:30 PM, Gregory Szorc <gregory.szorc@gmail.com> wrote:

> On 7/19/14, 3:25 PM, Matt Mackall wrote:
>> On Sat, 2014-07-19 at 15:00 -0500, Gregory Szorc wrote:
>>> I took the idea of this extension and built it out into a more finished
>>> implementation, which I'm calling "bundleclone."
>>> 
>>> https://hg.mozilla.org/hgcustom/version-control-tools/file/default/hgext/bundleclone
>>> 
>>> We may deploy this at Mozilla to reduce server load. More details at
>>> https://bugzilla.mozilla.org/show_bug.cgi?id=1041173
>>> 
>>> I'd appreciate any feedback.
>> 
>> Can I get you to make a wiki page for it?
> 
> http://mercurial.selenic.com/wiki/BundleCloneExtension
> 
> I also backported the extension to work with 2.5 and above.
> 

I have a few code review comments. How would you like them delivered?
Gregory Szorc - July 21, 2014, 2:20 p.m.
On 7/21/14, 8:42 AM, Augie Fackler wrote:
>
> On Jul 20, 2014, at 8:30 PM, Gregory Szorc <gregory.szorc@gmail.com> wrote:
>
>> On 7/19/14, 3:25 PM, Matt Mackall wrote:
>>> On Sat, 2014-07-19 at 15:00 -0500, Gregory Szorc wrote:
>>>> I took the idea of this extension and built it out into a more finished
>>>> implementation, which I'm calling "bundleclone."
>>>>
>>>> https://hg.mozilla.org/hgcustom/version-control-tools/file/default/hgext/bundleclone
>>>>
>>>> We may deploy this at Mozilla to reduce server load. More details at
>>>> https://bugzilla.mozilla.org/show_bug.cgi?id=1041173
>>>>
>>>> I'd appreciate any feedback.
>>>
>>> Can I get you to make a wiki page for it?
>>
>> http://mercurial.selenic.com/wiki/BundleCloneExtension
>>
>> I also backported the extension to work with 2.5 and above.
>>
>
> I have a few code review comments. How would you like them delivered?

Email to me is fine. I'll get a Bugzilla component created and update 
buglink in the extension sometime this week.
Augie Fackler - July 21, 2014, 2:21 p.m.
On Jul 21, 2014, at 10:20 AM, Gregory Szorc <gregory.szorc@gmail.com> wrote:

> On 7/21/14, 8:42 AM, Augie Fackler wrote:
>> 
>> On Jul 20, 2014, at 8:30 PM, Gregory Szorc <gregory.szorc@gmail.com> wrote:
>> 
>>> On 7/19/14, 3:25 PM, Matt Mackall wrote:
>>>> On Sat, 2014-07-19 at 15:00 -0500, Gregory Szorc wrote:
>>>>> I took the idea of this extension and built it out into a more finished
>>>>> implementation, which I'm calling "bundleclone."
>>>>> 
>>>>> https://hg.mozilla.org/hgcustom/version-control-tools/file/default/hgext/bundleclone
>>>>> 
>>>>> We may deploy this at Mozilla to reduce server load. More details at
>>>>> https://bugzilla.mozilla.org/show_bug.cgi?id=1041173
>>>>> 
>>>>> I'd appreciate any feedback.
>>>> 
>>>> Can I get you to make a wiki page for it?
>>> 
>>> http://mercurial.selenic.com/wiki/BundleCloneExtension
>>> 
>>> I also backported the extension to work with 2.5 and above.
>>> 
>> 
>> I have a few code review comments. How would you like them delivered?
> 
> Email to me is fine. I'll get a Bugzilla component created and update buglink in the extension sometime this week.

Will do. On second look-through, I don't have any immediate comments.

Patch

diff --git a/hgext/lookaside.py b/hgext/lookaside.py
new file mode 100644
--- /dev/null
+++ b/hgext/lookaside.py
@@ -0,0 +1,53 @@ 
+"""Hacky proof of concept for lookaside bundle.
+
+Right now this doesn't do anything clever about bundle generation or
+distribution. Put a bundle file (generated with 'hg bundle --all' at a
+URL that urllib2 can fetch, then put that url in
+.hg/durin42-lookaside-location.
+
+This might mulch your data somehow. It's relatively untested, but it
+proves the concept.
+"""
+
+import shutil
+import urllib2
+
+from mercurial import changegroup
+from mercurial import extensions
+from mercurial import wireproto
+
+def addcap(orig, *args, **kwargs):
+    c = orig(*args, **kwargs)
+    return c + ' x-durin42-lookaside'
+
+extensions.wrapfunction(wireproto, 'capabilities', addcap)
+wireproto.commands['capabilities'] = wireproto.capabilities, ''
+
+def lookaside(repo, proto):
+    try:
+        with repo.opener('durin42-lookaside-location') as f:
+            return f.read().strip()
+    except IOError:
+        return ''
+
+
+wireproto.commands['x-durin42-lookaside-clone'] = lookaside, ''
+
+def reposetup(ui, repo):
+    origcls = repo.__class__
+
+    class lookasiderepo(origcls):
+
+        def clone(self, remote, heads=[], stream=False):
+            if not heads and remote.capable('x-durin42-lookaside'):
+                url = remote._call('x-durin42-lookaside-clone')
+                if url:
+                    ui.debug('fetching lookaside bundle %s\n' % url)
+                    # TODO: support resuming the bundle stream if a connection fails
+                    self.addchangegroup(
+                        changegroup.readbundle(urllib2.urlopen(url), 'lookaside bundle'),
+                        'unbundle', 'bundle:lookaside bundle')
+                return self.pull(remote, heads)
+            return origcls.clone(self, remote, heads, stream=False)
+
+    repo.__class__ = lookasiderepo